From b4c28335b2613ccba794db5320fbe9ddd2806464 Mon Sep 17 00:00:00 2001 From: Tykayn Date: Thu, 10 Apr 2025 12:53:03 +0200 Subject: [PATCH] add tests and utils : limit values, get max in enums --- create_mapping.ts | 4 +- csv_to_geojson.ts | 2 +- makefile | 4 +- mappings/converters/configIRVE.ts | 6 ++- mappings/engine.ts | 51 ++++++++++++++----------- mappings/formatters.ts | 18 ++++----- mappings/mapping-config.type.ts | 1 + mappings/utils.ts | 56 +++++++++++++++++++++++++++- tests/main.test.js | 52 +++++++++++++++++++++++--- update_scripts/run_all_extractors.sh | 2 +- 10 files changed, 153 insertions(+), 43 deletions(-) diff --git a/create_mapping.ts b/create_mapping.ts index e60d396..46c3252 100644 --- a/create_mapping.ts +++ b/create_mapping.ts @@ -92,7 +92,7 @@ function updateMainFile(configName: string): void { let mainFileContent = fs.readFileSync(mainFilePath, 'utf8'); // Ajouter l'import - const importStatement = `import ${configName} from './mappings/converters/config${configName.replace(/^Mapping/, '')}'\n`; + const importStatement = `import ${configName} from './mappings/converters/config${configName.replaceAll(/^Mapping/, '')}'\n`; const importInsertPos = mainFileContent.indexOf('const limitWarningPercentageChangeInPoints'); if (importInsertPos !== -1) { @@ -143,7 +143,7 @@ async function main() { } // Générer le nom du fichier - const fileName = `config${configName.replace(/^Mapping/, '')}.ts`; + const fileName = `config${configName.replaceAll(/^Mapping/, '')}.ts`; const filePath = path.join(configDir, fileName); // Vérifier si le fichier existe déjà diff --git a/csv_to_geojson.ts b/csv_to_geojson.ts index 6a6bd0a..ae95a58 100644 --- a/csv_to_geojson.ts +++ b/csv_to_geojson.ts @@ -41,7 +41,7 @@ function csvToGeoJSON(options: Options): FeatureCollection { let headers: string[] = []; let headersFound = false; - let limitOffset = 30000000; + let limitOffset = 100000000; fs.createReadStream(filePath) .pipe(csvParser({ headers: hasHeaders })) diff --git a/makefile b/makefile index 8ef8a3e..175663a 100644 --- a/makefile +++ b/makefile @@ -33,4 +33,6 @@ osmose_irve: panneaux: npx ts-node unzip_csv.ts -u https://www.data.gouv.fr/fr/datasets/r/16a20f4e-0c06-40ff-adda-54f214099e5f -o panneaux_limite_de_vitesse_fr_panoramax_detections.csv -z npx ts-node csv_to_geojson.ts -d etalab_data/panneaux -f panneaux_limite_de_vitesse_fr_panoramax_detections.csv --latColumn 'GPSLatitude' --lonColumn 'GPSLongitude' -h - ts-node convert_to_osm_tags.ts --source etalab_data/panneaux/panneaux_limite_de_vitesse_fr_panoramax_detections.csv.geojson --output-file=panneaux_maxspeed_panoramax.json --engine-config=MappingPanneauxMaxSpeed \ No newline at end of file + ts-node convert_to_osm_tags.ts --source etalab_data/panneaux/panneaux_limite_de_vitesse_fr_panoramax_detections.csv.geojson --output-file=panneaux_maxspeed_panoramax.json --engine-config=MappingPanneauxMaxSpeed +tests: + npm test \ No newline at end of file diff --git a/mappings/converters/configIRVE.ts b/mappings/converters/configIRVE.ts index 44f46d9..371cfae 100644 --- a/mappings/converters/configIRVE.ts +++ b/mappings/converters/configIRVE.ts @@ -19,6 +19,7 @@ const MappingIRVE: MappingConfigType = { * select only certain points from the source */ filters: { + // offset: 1, // limiter à une feature pour faire des tests enable_coordinates_filter: false, enable_properties_filter: true, // filter_points_older_than_year: 2024, @@ -60,11 +61,12 @@ const MappingIRVE: MappingConfigType = { key_converted: 'operator:phone', convert_to_phone: true, // conversion en format international si possible }, - contact_operateur: 'operator:email', // ici, on souhaite convertir la clé contact_operateur=bidule en email=bidule + contact_operateur: 'operator:email', id_station_itinerance: { key_converted: 'ref:EU:EVSE', remove_stars: true, + truncate_enums_to_limit: 255, }, id_station_local: 'ref', @@ -84,6 +86,7 @@ const MappingIRVE: MappingConfigType = { }, reservation: { + key_converted: 'reservation', convert_to_boolean_value: true, // convertit en yes ou no }, // observations: 'note', @@ -125,6 +128,7 @@ const MappingIRVE: MappingConfigType = { }, // ******** champs plus complexes horaires: 'opening_hours', // déjà au bon format, enfin, en général. vérifier avec le validateur josm. + tarification: 'charge', paiement_cb: { key_converted: 'payment:credit_cards', // ignore_if_falsy: true, diff --git a/mappings/engine.ts b/mappings/engine.ts index ff550e8..25eae92 100644 --- a/mappings/engine.ts +++ b/mappings/engine.ts @@ -2,7 +2,7 @@ import custom_utils from './utils' import MappingConfigType from "./mapping-config.type"; import Formatters from "./formatters"; -const {debugLog} = custom_utils +const { debugLog, find_max_in_string, truncate_enums_to_limit } = custom_utils let listOfBooleanKeys = [ "prise_type_ef", @@ -57,7 +57,7 @@ export default class { let geoJSONConvertedPoint: any = {} - geoJSONConvertedPoint.properties = {...this.mapping_config.default_properties_of_point} + geoJSONConvertedPoint.properties = { ...this.mapping_config.default_properties_of_point } geoJSONConvertedPoint.type = featurePointGeoJson.type geoJSONConvertedPoint.geometry = featurePointGeoJson.geometry @@ -130,7 +130,7 @@ export default class { return newList; } - filterListOfPointsByExcludingIfColumnBeforeSomeYear(year: number, column:string, list_of_points: any[]): any[] { + filterListOfPointsByExcludingIfColumnBeforeSomeYear(year: number, column: string, list_of_points: any[]): any[] { let newList: any[] = [] list_of_points.forEach((geojsonPoint: any) => { let pointProperties = Object.keys(geojsonPoint.properties) @@ -138,7 +138,7 @@ export default class { // on inclut les points dont la date de création est de l'année demandée ou supérieure if (pointProperties.includes(column) && - (geojsonPoint.properties[column].substring(0,4) * 1) >= year + (geojsonPoint.properties[column].substring(0, 4) * 1) >= year ) { newList.push(geojsonPoint) } @@ -155,9 +155,12 @@ export default class { // trouver la valeur // socket_output_find_correspondances if (pointProperties.includes('puissance_nominale') && - 1 * (geojsonPoint.properties['puissance_nominale'].replace(' kW', '')) > minValue + 1 * (geojsonPoint.properties['puissance_nominale'].replaceAll(' kW', '')) > minValue ) { - newList.push(geojsonPoint) + let max_power = find_max_in_string(geojsonPoint.properties['puissance_nominale']) + if (max_power >= minValue) { + newList.push(geojsonPoint) + } } }) @@ -176,28 +179,25 @@ export default class { debugLog('mapElementFromConf: config_name', this.mapping_config.config_name) let mappingKeys = Object.keys(this.mapping_config.tags) - let featurePointPropertiesKeys = [] + let featurePointPropertiesKeys: string[] = []; if (this.mapping_config.osmose) { - // only creation of new points are handled by now [2023-10-07] - featurePointPropertiesKeys = Object.keys(featurePoint.properties.fixes[0][0].create) - // debugLog('featurePointPropertiesKeys', featurePointPropertiesKeys) - + featurePointPropertiesKeys = Object.keys(featurePoint.properties.fixes[0][0].create); } else { - featurePointPropertiesKeys = Object.keys(featurePoint.properties) + featurePointPropertiesKeys = Object.keys(featurePoint.properties); } debugLog('mapElementFromConf: ============= keys mappingKeys:', this.mapping_config.tags.length, mappingKeys.length) debugLog('mapElementFromConf: ============= keys featurePointPropertiesKeys :', featurePoint.properties.length, featurePointPropertiesKeys.length) - let newProperties = {...this.mapping_config.default_properties_of_point} + let newProperties = { ...this.mapping_config.default_properties_of_point } // reinit properties of current point let basePoint = Object.create(featurePoint) basePoint.type = featurePoint.type basePoint.geometry = featurePoint.geometry - basePoint.properties = {...this.mapping_config.default_properties_of_point} + basePoint.properties = { ...this.mapping_config.default_properties_of_point } // apply new properties if found in mapping config featurePointPropertiesKeys.forEach(pointKeyName => { @@ -279,6 +279,7 @@ export default class { let keyConvertedFromMapping = mappingKeys[mappingKeys.indexOf(pointKeyName)] let mappingConfigOfTag = this.mapping_config.tags[pointKeyName] + debugLog('========== mappingConfigOfTag', mappingConfigOfTag) debugLog('convertProperty: found element', pointKeyName, '=>', keyConvertedFromMapping, 'value : ', valueConvertedFromMapping) let convertedValue = originalValue @@ -387,13 +388,12 @@ export default class { } // ajouter les tags de socket newProperties - let converted_value = originalValue.replace(/[^\d\.\,]/g, '').replace(',', '.') + let converted_value = find_max_in_string(originalValue.replaceAll('.00', '').replaceAll(',', '.')) let max_output = 401 // do not limit accepted values let out = '' if (intOriginalValue < max_output) { - // rajouter l'unité de puissance kW dans la valeur out = converted_value + ' kW' @@ -402,13 +402,13 @@ export default class { debugLog('too high kW value detected', originalValue) if (intOriginalValue > 1000 && intOriginalValue < 401000) { - let kilowatts = (parseFloat(converted_value) / 1000).toFixed(2).replace('.00', ''); - out = ('' + kilowatts + ' kW').replace('.00', '') + let kilowatts = (converted_value / 1000).toFixed(2); + out = ('' + kilowatts + ' kW').replaceAll('.00', '') debugLog('valeurs en Watts out', out, 'original:', originalValue) this.stats.power_output++ } } - out = (out).replace('.00', '') + out = (out).replaceAll('.00', '') // debug land if (has_prise_type_combo_ccs) { @@ -453,6 +453,7 @@ export default class { debugLog('no sockets', this.current_geojson_point.properties.ref) } } + convertedValue = out; return out @@ -462,7 +463,8 @@ export default class { debugLog('invert boolean', convertedValue, originalValue) } if (configObject.remove_stars) { - convertedValue = originalValue.replace('*', '') + // Remplace toutes les occurrences de * de manière greedy + convertedValue = originalValue.replaceAll('*', '') debugLog('remove_stars', convertedValue, originalValue) } @@ -490,6 +492,13 @@ export default class { if (configObject.ignore_if_truthy && this.truthyValues.indexOf(originalValue) !== -1) { remove_original_key = true } + + if (configObject.truncate_enums_to_limit) { + // console.log('configObject.truncate_enums_to_limit', configObject) + convertedValue = custom_utils.truncate_enums_to_limit(convertedValue, configObject.truncate_enums_to_limit) + debugLog('truncate_enums_to_limit => ', convertedValue) + } + /** * config pour une clé * nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser @@ -570,7 +579,7 @@ export default class { debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}') debugLog(' =============== remove_original_key', newKey, remove_original_key) - let keysOfConfigObject = []; + let keysOfConfigObject: string[] = []; let hasKeyIgnoreThisData = false; if (configObject) { keysOfConfigObject = Object.keys(configObject) diff --git a/mappings/formatters.ts b/mappings/formatters.ts index 8308fd1..cc01192 100644 --- a/mappings/formatters.ts +++ b/mappings/formatters.ts @@ -1,26 +1,26 @@ import custom_utils from "./utils"; -const {debugLog,prefix_phone_fr_only} = custom_utils +const { debugLog, prefix_phone_fr_only } = custom_utils /** * Class that helps to convert values into predefined formats */ export default class Formatters { - static convertToPhone(originalValue: string) :string{ + static convertToPhone(originalValue: string): string { /** * nettoyer les numéros de téléphone en ne gardant que les nombres et le préfixe de pays */ - debugLog("convertToPhone:" , originalValue); + debugLog("convertToPhone:", originalValue); // debugLog('originalValue', originalValue.substring(1)) if (!originalValue) { originalValue = '' } - let original_without_spaces = originalValue.replace(' ', '') + let original_without_spaces = originalValue.replaceAll(' ', '') let cleaned_value = `${original_without_spaces}` cleaned_value = cleaned_value .trim() - .replace('Stations-e', '') + .replaceAll('Stations-e', '') .replace(/[a-zA-Zéèà]/ig, '') .replace(/[\(\)\.\- ]/g, '') let add_prefix = false; @@ -30,9 +30,9 @@ export default class Formatters { ) { add_prefix = true } - cleaned_value = cleaned_value.replace('+33', '') + cleaned_value = cleaned_value.replaceAll('+33', '') - debugLog("convertToPhone: cleaned_value" , cleaned_value); + debugLog("convertToPhone: cleaned_value", cleaned_value); if (/^0/.test(cleaned_value)) { cleaned_value = cleaned_value.substring(1) @@ -55,7 +55,7 @@ export default class Formatters { }) - convertedValue = convertedValue.replace(' ', ' ').trim(); + convertedValue = convertedValue.replaceAll(' ', ' ').trim(); debugLog('convertedValue', convertedValue) if ( /^\d/.test(convertedValue) && @@ -70,7 +70,7 @@ export default class Formatters { debugLog('phone: ', originalValue, '=>', convertedValue) - return ""+convertedValue; + return "" + convertedValue; } static convertToName(originalValue: string) { diff --git a/mappings/mapping-config.type.ts b/mappings/mapping-config.type.ts index c4949a2..f7ef160 100644 --- a/mappings/mapping-config.type.ts +++ b/mappings/mapping-config.type.ts @@ -74,6 +74,7 @@ export interface FeaturePropertyMappingConfigType { ignore_if_falsy?: boolean, ignore_if_truthy?: boolean, remove_stars?: boolean, + truncate_enums_to_limit?: number, conditional_values?: ConditionnalValuesConfigType, transform_function?: Function, diff --git a/mappings/utils.ts b/mappings/utils.ts index 04bee7b..cc34c7f 100644 --- a/mappings/utils.ts +++ b/mappings/utils.ts @@ -54,7 +54,7 @@ function writeFile(fileName: string, fileContent: any, outputPathOverride: strin console.log('pas de output', outputPathOverride ) } - let destination = `./${output_folder}/${fileName}`.replace('//', '/'); + let destination = `./${output_folder}/${fileName}`.replaceAll('//', '/'); console.log('write file ', destination) return fs.writeFile( destination, @@ -70,9 +70,61 @@ function writeFile(fileName: string, fileContent: any, outputPathOverride: strin ) } +/** + * trouve le max dans une string séparée par des points-virgules + * @param str + * @returns + */ +function find_max_in_string(str: string, separator: string = ';') { + let max = 0; + if (str.indexOf(separator) !== -1) { + let explode = str.split(separator); + explode.forEach((item: string) => { + // keep only the number + let value = parseInt(item.replaceAll(/[^0-9]/g, '')); + if (value && value > max) { + max = value; + } + }); + return max; + } + return parseInt(str.replaceAll(/[^0-9]/g, '')); +} + +/** + * tronque une string à une longueur donnée + * @param str + * @param limit + * @returns + */ +function truncate_enums_to_limit(str: string, limit: number = 255) { + if (str.indexOf(';') !== -1) { + let elements = str.split(';'); + let result: string[] = []; + let currentLength = 0; + + for (let element of elements) { + // +1 pour le point-virgule qui sera ajouté + if (currentLength + element.length + 1 <= limit) { + result.push(element); + currentLength += element.length + 1; + } + + } + let joined = result.join(';'); + return joined; + } + if (str.length > limit) { + return str.substring(0, limit) + } + return str +} + export default { debugLog, isBooleanKey, writeFile, - prefix_phone_fr_only + find_max_in_string, + truncate_enums_to_limit, + prefix_phone_fr_only, } diff --git a/tests/main.test.js b/tests/main.test.js index 02bdf3f..38b3b9a 100644 --- a/tests/main.test.js +++ b/tests/main.test.js @@ -6,6 +6,8 @@ import { mappingName, mappingSame, mappingTruthy, mappingFalsy, mappingIgnoreFalsy, mappingIgnoreTruthy } from './data/mappings_to_test' +import utils from '../mappings/utils' +import Formatters from '../mappings/formatters' const testingGeoJson = require('./data/testing.json') @@ -29,10 +31,10 @@ describe('mapping properties with rich mapping engine', () => { let newProperties = Mapping_engine.convertProperty('equal', Object.keys(mappingSame.tags), feature_to_test, - mappingSame.default_properties_of_point ) + mappingSame.default_properties_of_point) expect(newProperties).toStrictEqual({ - equal : "same value" + equal: "same value" }) }) test('retrieve config name in mapping engine', () => { @@ -41,11 +43,11 @@ describe('mapping properties with rich mapping engine', () => { }) test('maps nom_amenageur to name, and keep the same value', () => { let Mapping_engine = new mapping_engine(mappingName) - let newProperties = Mapping_engine.convertProperty('nom_amenageur',Object.keys(mappingName.tags),feature_to_test,mappingName.default_properties_of_point ) + let newProperties = Mapping_engine.convertProperty('nom_amenageur', Object.keys(mappingName.tags), feature_to_test, mappingName.default_properties_of_point) expect(Mapping_engine.getConfig().config_name).toBe('testing config mappingName') expect(newProperties).toStrictEqual({ - name : "Bob" + name: "Bob" }) }) test('ignore one value if it is truthy', () => { @@ -123,7 +125,7 @@ describe('mapping properties with rich mapping engine', () => { feature_to_test.properties.telephone_operateur = 'Stations-e' mapped_point = Mapping_engine.mapElementFromConf(feature_to_test) - expect(mapped_point.properties).toStrictEqual({ }) + expect(mapped_point.properties).toStrictEqual({}) feature_to_test.properties.telephone_operateur = '+33 0 7 66 38 74 96' expected_converted_phone = '+33 7 66 38 74 96' @@ -162,3 +164,43 @@ xdescribe('infer domain of values from geojson file', () => { xdescribe('build mapping engine config from unique values', () => { test('builds a valid mapping config', () => { }) }) + +describe('find max in enum', () => { + test('value has enums', () => { + let value = '1;2;3;10;4;5;6;7;8;9;' + let max = utils.find_max_in_string(value) + expect(max).toBe(10) + }) + test('value has enums with maybe units', () => { + let value = '1;2;3;10 km;4;5 kw;6;7;8;9;' + let max = utils.find_max_in_string(value) + expect(max).toBe(10) + }) + + test('value has no enums', () => { + let max = utils.find_max_in_string('10') + console.log('max', max) + expect(max).toBe(10) + }) +}) + +describe('truncate enums to limit', () => { + test('truncate enums to limit', () => { + let value = '1;2;3;10 km;4;5 kw;6;7;8;9;' + let truncated = utils.truncate_enums_to_limit(value, 5) + expect(truncated).toBe('1;2;') + }) +}) + +describe('formatters tests', () => { + test('format phone', () => { + let phone = '+331 2345 6789' + let formatted = Formatters.convertToPhone(phone) + expect(formatted).toBe('+33 1 23 45 67 89') + }) + test('format name', () => { + let name = 'bob lénon' + let formatted = Formatters.convertToName(name) + expect(formatted).toBe('Bob lénon') + }) +}) \ No newline at end of file diff --git a/update_scripts/run_all_extractors.sh b/update_scripts/run_all_extractors.sh index 26fa85f..291edf8 100755 --- a/update_scripts/run_all_extractors.sh +++ b/update_scripts/run_all_extractors.sh @@ -2,7 +2,7 @@ # chemin du dossier à parcourir source functions.sh -dir_to_search="../mappings/extractors" +dir_to_search="mappings/extractors" # recherche tous les fichiers .sh dans le dossier et ses sous-dossiers find "$dir_to_search" -type f -name "*.sh" -print0 | while IFS= read -r -d '' file; do