add tests and utils : limit values, get max in enums

create_mapping.ts

@@ -92,7 +92,7 @@ function updateMainFile(configName: string): void {
     let mainFileContent = fs.readFileSync(mainFilePath, 'utf8');
 
     // Ajouter l'import
-    const importStatement = `import ${configName} from './mappings/converters/config${configName.replace(/^Mapping/, '')}'\n`;
+    const importStatement = `import ${configName} from './mappings/converters/config${configName.replaceAll(/^Mapping/, '')}'\n`;
     const importInsertPos = mainFileContent.indexOf('const limitWarningPercentageChangeInPoints');
 
     if (importInsertPos !== -1) {
@@ -143,7 +143,7 @@ async function main() {
   }
 
   // Générer le nom du fichier
-  const fileName = `config${configName.replace(/^Mapping/, '')}.ts`;
+  const fileName = `config${configName.replaceAll(/^Mapping/, '')}.ts`;
   const filePath = path.join(configDir, fileName);
 
   // Vérifier si le fichier existe déjà

csv_to_geojson.ts

@@ -41,7 +41,7 @@ function csvToGeoJSON(options: Options): FeatureCollection<Point> {
   let headers: string[] = [];
   let headersFound = false;
 
-  let limitOffset = 30000000;
+  let limitOffset = 100000000;
 
   fs.createReadStream(filePath)
     .pipe(csvParser({ headers: hasHeaders }))

makefile

@@ -34,3 +34,5 @@ panneaux:
 	npx ts-node unzip_csv.ts -u https://www.data.gouv.fr/fr/datasets/r/16a20f4e-0c06-40ff-adda-54f214099e5f -o panneaux_limite_de_vitesse_fr_panoramax_detections.csv -z
 	npx ts-node csv_to_geojson.ts -d etalab_data/panneaux -f panneaux_limite_de_vitesse_fr_panoramax_detections.csv --latColumn 'GPSLatitude' --lonColumn 'GPSLongitude' -h
 	ts-node convert_to_osm_tags.ts --source etalab_data/panneaux/panneaux_limite_de_vitesse_fr_panoramax_detections.csv.geojson --output-file=panneaux_maxspeed_panoramax.json --engine-config=MappingPanneauxMaxSpeed
+tests:
+	npm test

mappings/converters/configIRVE.ts

@@ -19,6 +19,7 @@ const MappingIRVE: MappingConfigType = {
      * select only certain points from the source
      */
     filters: {
+        // offset: 1, // limiter à une feature pour faire des tests
         enable_coordinates_filter: false,
         enable_properties_filter: true,
         // filter_points_older_than_year: 2024,
@@ -60,11 +61,12 @@ const MappingIRVE: MappingConfigType = {
             key_converted: 'operator:phone',
             convert_to_phone: true, // conversion en format international si possible
         },
-        contact_operateur: 'operator:email', // ici, on souhaite convertir la clé contact_operateur=bidule en email=bidule
+        contact_operateur: 'operator:email',
 
         id_station_itinerance: {
             key_converted: 'ref:EU:EVSE',
             remove_stars: true,
+            truncate_enums_to_limit: 255,
         },
 
         id_station_local: 'ref',
@@ -84,6 +86,7 @@ const MappingIRVE: MappingConfigType = {
 
         },
         reservation: {
+            key_converted: 'reservation',
             convert_to_boolean_value: true, // convertit en yes ou no
         },
         // observations: 'note',
@@ -125,6 +128,7 @@ const MappingIRVE: MappingConfigType = {
         },
         // ******** champs plus complexes
         horaires: 'opening_hours', // déjà au bon format, enfin, en général. vérifier avec le validateur josm.
+        tarification: 'charge',
         paiement_cb: {
             key_converted: 'payment:credit_cards',
             // ignore_if_falsy: true,

mappings/engine.ts

@@ -2,7 +2,7 @@ import custom_utils from './utils'
 import MappingConfigType from "./mapping-config.type";
 import Formatters from "./formatters";
 
-const {debugLog} = custom_utils
+const { debugLog, find_max_in_string, truncate_enums_to_limit } = custom_utils
 
 let listOfBooleanKeys = [
     "prise_type_ef",
@@ -57,7 +57,7 @@ export default class {
         let geoJSONConvertedPoint: any = {}
 
 
-        geoJSONConvertedPoint.properties = {...this.mapping_config.default_properties_of_point}
+        geoJSONConvertedPoint.properties = { ...this.mapping_config.default_properties_of_point }
         geoJSONConvertedPoint.type = featurePointGeoJson.type
         geoJSONConvertedPoint.geometry = featurePointGeoJson.geometry
 
@@ -130,7 +130,7 @@ export default class {
         return newList;
     }
 
-    filterListOfPointsByExcludingIfColumnBeforeSomeYear(year: number, column:string, list_of_points: any[]): any[] {
+    filterListOfPointsByExcludingIfColumnBeforeSomeYear(year: number, column: string, list_of_points: any[]): any[] {
         let newList: any[] = []
         list_of_points.forEach((geojsonPoint: any) => {
             let pointProperties = Object.keys(geojsonPoint.properties)
@@ -138,7 +138,7 @@ export default class {
 
             // on inclut les points dont la date de création est de l'année demandée ou supérieure
             if (pointProperties.includes(column) &&
-                (geojsonPoint.properties[column].substring(0,4) * 1) >= year
+                (geojsonPoint.properties[column].substring(0, 4) * 1) >= year
             ) {
                 newList.push(geojsonPoint)
             }
@@ -155,9 +155,12 @@ export default class {
             // trouver la valeur
             // socket_output_find_correspondances
             if (pointProperties.includes('puissance_nominale') &&
-                1 * (geojsonPoint.properties['puissance_nominale'].replace(' kW', '')) > minValue
+                1 * (geojsonPoint.properties['puissance_nominale'].replaceAll(' kW', '')) > minValue
             ) {
-                newList.push(geojsonPoint)
+                let max_power = find_max_in_string(geojsonPoint.properties['puissance_nominale'])
+                if (max_power >= minValue) {
+                    newList.push(geojsonPoint)
+                }
             }
         })
 
@@ -176,28 +179,25 @@ export default class {
 
         debugLog('mapElementFromConf: config_name', this.mapping_config.config_name)
         let mappingKeys = Object.keys(this.mapping_config.tags)
-        let featurePointPropertiesKeys = []
+        let featurePointPropertiesKeys: string[] = [];
         if (this.mapping_config.osmose) {
-            // only creation of new points are handled by now [2023-10-07]
+            featurePointPropertiesKeys = Object.keys(featurePoint.properties.fixes[0][0].create);
-            featurePointPropertiesKeys = Object.keys(featurePoint.properties.fixes[0][0].create)
-            // debugLog('featurePointPropertiesKeys', featurePointPropertiesKeys)
-
         } else {
-            featurePointPropertiesKeys = Object.keys(featurePoint.properties)
+            featurePointPropertiesKeys = Object.keys(featurePoint.properties);
         }
 
 
         debugLog('mapElementFromConf: ============= keys mappingKeys:', this.mapping_config.tags.length, mappingKeys.length)
         debugLog('mapElementFromConf: ============= keys featurePointPropertiesKeys :', featurePoint.properties.length, featurePointPropertiesKeys.length)
 
-        let newProperties = {...this.mapping_config.default_properties_of_point}
+        let newProperties = { ...this.mapping_config.default_properties_of_point }
 
 
         // reinit properties of current point
         let basePoint = Object.create(featurePoint)
         basePoint.type = featurePoint.type
         basePoint.geometry = featurePoint.geometry
-        basePoint.properties = {...this.mapping_config.default_properties_of_point}
+        basePoint.properties = { ...this.mapping_config.default_properties_of_point }
 
         // apply new properties if found in mapping config
         featurePointPropertiesKeys.forEach(pointKeyName => {
@@ -279,6 +279,7 @@ export default class {
                 let keyConvertedFromMapping = mappingKeys[mappingKeys.indexOf(pointKeyName)]
                 let mappingConfigOfTag = this.mapping_config.tags[pointKeyName]
 
+
                 debugLog('========== mappingConfigOfTag', mappingConfigOfTag)
                 debugLog('convertProperty: found element', pointKeyName, '=>', keyConvertedFromMapping, 'value : ', valueConvertedFromMapping)
                 let convertedValue = originalValue
@@ -387,13 +388,12 @@ export default class {
                         }
                         //  ajouter les tags de socket newProperties
 
-                        let converted_value = originalValue.replace(/[^\d\.\,]/g, '').replace(',', '.')
+                        let converted_value = find_max_in_string(originalValue.replaceAll('.00', '').replaceAll(',', '.'))
                         let max_output = 401
                         // do not limit accepted values
                         let out = ''
 
                         if (intOriginalValue < max_output) {
-
                             // rajouter l'unité de puissance kW dans la valeur
                             out = converted_value + ' kW'
 
@@ -402,13 +402,13 @@ export default class {
                             debugLog('too high kW value detected', originalValue)
                             if (intOriginalValue > 1000 && intOriginalValue < 401000) {
 
-                                let kilowatts = (parseFloat(converted_value) / 1000).toFixed(2).replace('.00', '');
+                                let kilowatts = (converted_value / 1000).toFixed(2);
-                                out = ('' + kilowatts + ' kW').replace('.00', '')
+                                out = ('' + kilowatts + ' kW').replaceAll('.00', '')
                                 debugLog('valeurs en Watts out', out, 'original:', originalValue)
                                 this.stats.power_output++
                             }
                         }
-                        out = (out).replace('.00', '')
+                        out = (out).replaceAll('.00', '')
 
                         // debug land
                         if (has_prise_type_combo_ccs) {
@@ -453,6 +453,7 @@ export default class {
                                 debugLog('no sockets', this.current_geojson_point.properties.ref)
                             }
                         }
+                        convertedValue = out;
                         return out
 
 
@@ -462,7 +463,8 @@ export default class {
                         debugLog('invert boolean', convertedValue, originalValue)
                     }
                     if (configObject.remove_stars) {
-                        convertedValue = originalValue.replace('*', '')
+                        // Remplace toutes les occurrences de * de manière greedy
+                        convertedValue = originalValue.replaceAll('*', '')
                         debugLog('remove_stars', convertedValue, originalValue)
                     }
 
@@ -490,6 +492,13 @@ export default class {
                     if (configObject.ignore_if_truthy && this.truthyValues.indexOf(originalValue) !== -1) {
                         remove_original_key = true
                     }
+
+                    if (configObject.truncate_enums_to_limit) {
+                        // console.log('configObject.truncate_enums_to_limit', configObject)
+                        convertedValue = custom_utils.truncate_enums_to_limit(convertedValue, configObject.truncate_enums_to_limit)
+                        debugLog('truncate_enums_to_limit => ', convertedValue)
+                    }
+
                     /**
                      * config pour une clé
                      * nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser
@@ -570,7 +579,7 @@ export default class {
                     debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}')
                     debugLog(' =============== remove_original_key', newKey, remove_original_key)
 
-                    let keysOfConfigObject = [];
+                    let keysOfConfigObject: string[] = [];
                     let hasKeyIgnoreThisData = false;
                     if (configObject) {
                         keysOfConfigObject = Object.keys(configObject)

mappings/formatters.ts

@@ -1,26 +1,26 @@
 import custom_utils from "./utils";
 
-const {debugLog,prefix_phone_fr_only} = custom_utils
+const { debugLog, prefix_phone_fr_only } = custom_utils
 
 /**
  * Class that helps to convert values into predefined formats
  */
 export default class Formatters {
 
-    static convertToPhone(originalValue: string) :string{
+    static convertToPhone(originalValue: string): string {
         /**
          * nettoyer les numéros de téléphone en ne gardant que les nombres et le préfixe de pays
          */
-        debugLog("convertToPhone:" , originalValue);
+        debugLog("convertToPhone:", originalValue);
         // debugLog('originalValue', originalValue.substring(1))
         if (!originalValue) {
             originalValue = ''
         }
-        let original_without_spaces = originalValue.replace(' ', '')
+        let original_without_spaces = originalValue.replaceAll(' ', '')
         let cleaned_value = `${original_without_spaces}`
         cleaned_value = cleaned_value
             .trim()
-            .replace('Stations-e', '')
+            .replaceAll('Stations-e', '')
             .replace(/[a-zA-Zéèà]/ig, '')
             .replace(/[\(\)\.\- ]/g, '')
         let add_prefix = false;
@@ -30,9 +30,9 @@ export default class Formatters {
         ) {
             add_prefix = true
         }
-        cleaned_value = cleaned_value.replace('+33', '')
+        cleaned_value = cleaned_value.replaceAll('+33', '')
 
-        debugLog("convertToPhone: cleaned_value" , cleaned_value);
+        debugLog("convertToPhone: cleaned_value", cleaned_value);
 
         if (/^0/.test(cleaned_value)) {
             cleaned_value = cleaned_value.substring(1)
@@ -55,7 +55,7 @@ export default class Formatters {
         })
 
 
-        convertedValue = convertedValue.replace('  ', ' ').trim();
+        convertedValue = convertedValue.replaceAll('  ', ' ').trim();
         debugLog('convertedValue', convertedValue)
         if (
             /^\d/.test(convertedValue) &&
@@ -70,7 +70,7 @@ export default class Formatters {
         debugLog('phone: ', originalValue, '=>', convertedValue)
 
 
-        return ""+convertedValue;
+        return "" + convertedValue;
     }
 
     static convertToName(originalValue: string) {

mappings/mapping-config.type.ts

@@ -74,6 +74,7 @@ export interface FeaturePropertyMappingConfigType {
     ignore_if_falsy?: boolean,
     ignore_if_truthy?: boolean,
     remove_stars?: boolean,
+    truncate_enums_to_limit?: number,
     conditional_values?: ConditionnalValuesConfigType,
     transform_function?: Function,
 

mappings/utils.ts

@@ -54,7 +54,7 @@ function writeFile(fileName: string, fileContent: any, outputPathOverride: strin
         console.log('pas de output', outputPathOverride
         )
     }
-    let destination = `./${output_folder}/${fileName}`.replace('//', '/');
+    let destination = `./${output_folder}/${fileName}`.replaceAll('//', '/');
     console.log('write file ', destination)
     return fs.writeFile(
         destination,
@@ -70,9 +70,61 @@ function writeFile(fileName: string, fileContent: any, outputPathOverride: strin
     )
 }
 
+/**
+ * trouve le max dans une string séparée par des points-virgules
+ * @param str
+ * @returns
+ */
+function find_max_in_string(str: string, separator: string = ';') {
+    let max = 0;
+    if (str.indexOf(separator) !== -1) {
+        let explode = str.split(separator);
+        explode.forEach((item: string) => {
+            // keep only the number
+            let value = parseInt(item.replaceAll(/[^0-9]/g, ''));
+            if (value && value > max) {
+                max = value;
+            }
+        });
+        return max;
+    }
+    return parseInt(str.replaceAll(/[^0-9]/g, ''));
+}
+
+/**
+ * tronque une string à une longueur donnée
+ * @param str
+ * @param limit
+ * @returns
+ */
+function truncate_enums_to_limit(str: string, limit: number = 255) {
+    if (str.indexOf(';') !== -1) {
+        let elements = str.split(';');
+        let result: string[] = [];
+        let currentLength = 0;
+
+        for (let element of elements) {
+            // +1 pour le point-virgule qui sera ajouté
+            if (currentLength + element.length + 1 <= limit) {
+                result.push(element);
+                currentLength += element.length + 1;
+            }
+
+        }
+        let joined = result.join(';');
+        return joined;
+    }
+    if (str.length > limit) {
+        return str.substring(0, limit)
+    }
+    return str
+}
+
 export default {
     debugLog,
     isBooleanKey,
     writeFile,
-    prefix_phone_fr_only
+    find_max_in_string,
+    truncate_enums_to_limit,
+    prefix_phone_fr_only,
 }

tests/main.test.js

@@ -6,6 +6,8 @@ import {
 	mappingName,
 	mappingSame, mappingTruthy, mappingFalsy, mappingIgnoreFalsy, mappingIgnoreTruthy
 } from './data/mappings_to_test'
+import utils from '../mappings/utils'
+import Formatters from '../mappings/formatters'
 
 const testingGeoJson = require('./data/testing.json')
 
@@ -29,10 +31,10 @@ describe('mapping properties with rich mapping engine', () => {
 		let newProperties = Mapping_engine.convertProperty('equal',
 			Object.keys(mappingSame.tags),
 			feature_to_test,
-			mappingSame.default_properties_of_point )
+			mappingSame.default_properties_of_point)
 
 		expect(newProperties).toStrictEqual({
-			equal : "same value"
+			equal: "same value"
 		})
 	})
 	test('retrieve config name in mapping engine', () => {
@@ -41,11 +43,11 @@ describe('mapping properties with rich mapping engine', () => {
 	})
 	test('maps nom_amenageur to name, and keep the same value', () => {
 		let Mapping_engine = new mapping_engine(mappingName)
-		let newProperties = Mapping_engine.convertProperty('nom_amenageur',Object.keys(mappingName.tags),feature_to_test,mappingName.default_properties_of_point )
+		let newProperties = Mapping_engine.convertProperty('nom_amenageur', Object.keys(mappingName.tags), feature_to_test, mappingName.default_properties_of_point)
 		expect(Mapping_engine.getConfig().config_name).toBe('testing config mappingName')
 
 		expect(newProperties).toStrictEqual({
-			name : "Bob"
+			name: "Bob"
 		})
 	})
 	test('ignore one value if it is truthy', () => {
@@ -123,7 +125,7 @@ describe('mapping properties with rich mapping engine', () => {
 
 		feature_to_test.properties.telephone_operateur = 'Stations-e'
 		mapped_point = Mapping_engine.mapElementFromConf(feature_to_test)
-		expect(mapped_point.properties).toStrictEqual({ })
+		expect(mapped_point.properties).toStrictEqual({})
 
 		feature_to_test.properties.telephone_operateur = '+33 0 7 66 38 74 96'
 		expected_converted_phone = '+33 7 66 38 74 96'
@@ -162,3 +164,43 @@ xdescribe('infer domain of values from geojson file', () => {
 xdescribe('build mapping engine config from unique values', () => {
 	test('builds a valid mapping config', () => { })
 })
+
+describe('find max in enum', () => {
+	test('value has enums', () => {
+		let value = '1;2;3;10;4;5;6;7;8;9;'
+		let max = utils.find_max_in_string(value)
+		expect(max).toBe(10)
+	})
+	test('value has enums with maybe units', () => {
+		let value = '1;2;3;10 km;4;5 kw;6;7;8;9;'
+		let max = utils.find_max_in_string(value)
+		expect(max).toBe(10)
+	})
+
+	test('value has no enums', () => {
+		let max = utils.find_max_in_string('10')
+		console.log('max', max)
+		expect(max).toBe(10)
+	})
+})
+
+describe('truncate enums to limit', () => {
+	test('truncate enums to limit', () => {
+		let value = '1;2;3;10 km;4;5 kw;6;7;8;9;'
+		let truncated = utils.truncate_enums_to_limit(value, 5)
+		expect(truncated).toBe('1;2;')
+	})
+})
+
+describe('formatters tests', () => {
+	test('format phone', () => {
+		let phone = '+331 2345 6789'
+		let formatted = Formatters.convertToPhone(phone)
+		expect(formatted).toBe('+33 1 23 45 67 89')
+	})
+	test('format name', () => {
+		let name = 'bob lénon'
+		let formatted = Formatters.convertToName(name)
+		expect(formatted).toBe('Bob lénon')
+	})
+})

update_scripts/run_all_extractors.sh

@@ -2,7 +2,7 @@
 
 # chemin du dossier à parcourir
 source functions.sh
-dir_to_search="../mappings/extractors"
+dir_to_search="mappings/extractors"
 
 # recherche tous les fichiers .sh dans le dossier et ses sous-dossiers
 find "$dir_to_search" -type f -name "*.sh" -print0 | while IFS= read -r -d '' file; do