From a732edc22814398d9682a1857d53f1e4bc16d149 Mon Sep 17 00:00:00 2001 From: Tykayn Date: Fri, 11 Apr 2025 15:53:23 +0200 Subject: [PATCH] split utils, separate IRVE guess --- config.ts | 73 ++++++++ convert_to_osm_tags.ts | 87 ++------- create_mapping.ts | 4 +- csv_to_geojson.ts | 159 ++-------------- csv_to_geojson.utils.ts | 153 +++++++++++++++ mappings/converters/configIRVE.ts | 6 +- mappings/converters/configPlanningFamilial.ts | 22 ++- mappings/engine.ts | 175 +++++------------- mappings/irve.utils.ts | 109 +++++++++++ mappings/mapping-config.type.ts | 1 + mappings/utils.ts | 29 ++- tests/main.test.js | 5 + 12 files changed, 460 insertions(+), 363 deletions(-) create mode 100644 config.ts create mode 100644 csv_to_geojson.utils.ts create mode 100644 mappings/irve.utils.ts diff --git a/config.ts b/config.ts new file mode 100644 index 0000000..dec25b1 --- /dev/null +++ b/config.ts @@ -0,0 +1,73 @@ +/** + * config de Wololo + * ----------------- + */ + +// déclaration des mappings, un par thématique de données +// vous pouvez mélanger plusieurs jeux de données dans un mapping, mais faites attention aux propriétés nommées identiques + +import mappingRouenParkingVelos from "./mappings/converters/configRouen_OpenData_velo_parkings"; +import mappingFINESS from "./mappings/converters/configFINESS"; +import MappingArbresIssy from "./mappings/converters/configArbresIssy"; +import MappingArbresEchirolles from "./mappings/converters/configArbresEchirolles"; +import MappingArbres92 from "./mappings/converters/configArbresHautsDeSeine"; +import MappingMuseums from "./mappings/converters/configMuseums"; +import MappingRouenPAV from "./mappings/converters/configRouen_PAV"; +import MappingAskAngela from "./mappings/converters/configAskAngela"; +import MappingPlanningFamlial from "./mappings/converters/configPlanningFamilial"; +import MappingSurveillanceRouen from "./mappings/converters/configSurveillance"; +import MappingRnb from "./mappings/converters/configRnb"; +import MappingArbresRemarquablesRouen from './mappings/converters/configArbresRemarquablesRouen' +import MappingPanneauxMaxSpeed from './mappings/converters/configPanneauxMaxSpeed' +import mappingConfigIRVE from './mappings/converters/configIRVE' +import ConfigIRVE from './mappings/converters/configIRVE' +import mappingIssy2Roues from './mappings/converters/configIssy_OpenData_2roues' +import mappingConfigIRVEFromOsmose from './mappings/converters/configIRVE_osmose' +import mappingConfigIRVE_simple from './mappings/converters/mappingConfigIRVE_simple' +import mappingTest from './mappings/converters/configTest' + +const allowed_configs: any = { + MappingPanneauxMaxSpeed, + MappingArbresRemarquablesRouen, + MappingRnb, + mappingIssy2Roues, + mappingConfigIRVE, + mappingConfigIRVEFromOsmose, + mappingConfigIRVE_simple, + mappingTest, + ConfigIRVE, + mappingRouenParkingVelos, + mappingFINESS, + MappingArbresIssy, + MappingArbresEchirolles, + MappingArbres92, + MappingMuseums, + MappingRouenPAV, + MappingAskAngela, + MappingPlanningFamlial, + MappingSurveillanceRouen +}; + +/** + * clés booléennes + */ +let listOfBooleanKeys = [ + "prise_type_ef", + "prise_type_2", + "prise_type_combo_ccs", + "prise_type_chademo", + "gratuit", + "paiement_acte", + "paiement_cb", + "cable_t2_attache" +] +let irve_max_output = 401 // limite de kW de puissance pour une borne de recharge publique + +const limitWarningPercentageChangeInPoints = 5; // show a warning when more than N percent of the number of points changed + +export default { + listOfBooleanKeys, + irve_max_output, + allowed_configs, + limitWarningPercentageChangeInPoints +} \ No newline at end of file diff --git a/convert_to_osm_tags.ts b/convert_to_osm_tags.ts index 0d881d0..2287fe4 100644 --- a/convert_to_osm_tags.ts +++ b/convert_to_osm_tags.ts @@ -7,24 +7,7 @@ * --osmose : Active le format Osmose pour la conversion * --source=chemin/vers/fichier.json : Spécifie le fichier source à convertir * --engine=true|false : Active/désactive le moteur de mapping - * --engine-config=NomConfig : Utilise une configuration spécifique parmi: - * - MappingRnb - * - mappingIssy2Roues - * - mappingConfigIRVE - * - mappingConfigIRVEFromOsmose - * - mappingConfigIRVE_simple - * - mappingTest - * - ConfigIRVE - * - mappingRouenParkingVelos - * - mappingFINESS - * - MappingArbresIssy - * - MappingArbresEchirolles - * - MappingArbres92 - * - MappingMuseums - * - MappingRouenPAV - * - MappingAskAngela - * - MappingPlanningFamlial - * - MappingSurveillanceRouen + * --engine-config=NomConfig : Utilise une configuration spécifique parmi celles déclarées * --output-file=nom_fichier : Spécifie le nom du fichier de sortie * --outname=nom_fichier : Alias pour --output-file * --testingConfig : Active le mode test avec la configuration mappingTest @@ -33,51 +16,12 @@ import * as fs from 'fs' -import mappingConfigIRVE from './mappings/converters/configIRVE' -import ConfigIRVE from './mappings/converters/configIRVE' -import mappingIssy2Roues from './mappings/converters/configIssy_OpenData_2roues' -import mappingConfigIRVEFromOsmose from './mappings/converters/configIRVE_osmose' -import mappingConfigIRVE_simple from './mappings/converters/mappingConfigIRVE_simple' -import mappingTest from './mappings/converters/configTest' import mapping_engine from './mappings/engine' import MappingConfigType, { BoundingBoxCoordinatesType, FeatureCollection } from "./mappings/mapping-config.type"; import utils from './mappings/utils' -import mappingRouenParkingVelos from "./mappings/converters/configRouen_OpenData_velo_parkings"; -import mappingFINESS from "./mappings/converters/configFINESS"; -import MappingArbresIssy from "./mappings/converters/configArbresIssy"; -import MappingArbresEchirolles from "./mappings/converters/configArbresEchirolles"; -import MappingArbres92 from "./mappings/converters/configArbresHautsDeSeine"; -import MappingMuseums from "./mappings/converters/configMuseums"; -import MappingRouenPAV from "./mappings/converters/configRouen_PAV"; -import MappingAskAngela from "./mappings/converters/configAskAngela"; -import MappingPlanningFamlial from "./mappings/converters/configPlanningFamilial"; -import MappingSurveillanceRouen from "./mappings/converters/configSurveillance"; -import MappingRnb from "./mappings/converters/configRnb"; -import MappingArbresRemarquablesRouen from './mappings/converters/configArbresRemarquablesRouen' -import MappingPanneauxMaxSpeed from './mappings/converters/configPanneauxMaxSpeed' -const limitWarningPercentageChangeInPoints = 5; // show a warning when more than N percent of the number of points changed +import config from './config' + -const allowed_configs: any = { - MappingPanneauxMaxSpeed, - MappingArbresRemarquablesRouen, - MappingRnb, - mappingIssy2Roues, - mappingConfigIRVE, - mappingConfigIRVEFromOsmose, - mappingConfigIRVE_simple, - mappingTest, - ConfigIRVE, - mappingRouenParkingVelos, - mappingFINESS, - MappingArbresIssy, - MappingArbresEchirolles, - MappingArbres92, - MappingMuseums, - MappingRouenPAV, - MappingAskAngela, - MappingPlanningFamlial, - MappingSurveillanceRouen -}; const minimist = require('minimist'); @@ -138,12 +82,12 @@ if (mini_arguments['outname']) { } if (mini_arguments['testingConfig']) { console.log('testing') - Mapping_engine = new mapping_engine(mappingTest) + Mapping_engine = new mapping_engine(config.allowed_configs.mappingTest) } else if (osmoseFormat) { console.log(' *********** we use osmose converter *********') - Mapping_engine = new mapping_engine(mappingConfigIRVEFromOsmose) + Mapping_engine = new mapping_engine(config.allowed_configs.mappingConfigIRVEFromOsmose) } else { - Mapping_engine = new mapping_engine(mappingConfigIRVE) + Mapping_engine = new mapping_engine(config.allowed_configs.mappingConfigIRVE) } let filterZipCode = new RegExp(`^${filterDepartment}`) @@ -416,7 +360,7 @@ function convertDataFromSource(sourceFilePath: string, mapping: MappingConfigTyp console.log('Changement de features', percentChange, '%') - if (percentChange > limitWarningPercentageChangeInPoints) { + if (percentChange > config.limitWarningPercentageChangeInPoints) { console.log(' /!\\ pas mal de points en moins, plus de ' + percentChange + '%') } debugLog('convert : write file ', fileNameToWrite) @@ -480,22 +424,22 @@ function mapElementFromConfSimple(featurePoint: any, mappingConfig: any) { } function setMappingConfigFromName(engine_conf_choice: string) { - console.log('------- sourceFilePathGeoJson', sourceFilePathGeoJson) - console.log('------- engine_conf_choice', engine_conf_choice) + console.log('------- geojson source', sourceFilePathGeoJson) + console.log('------- conversion engine', engine_conf_choice) if (use_mapping_engine) { debugLog(' - using mapping engine') debugLog(' - pointCounterMax', pointCounterMax) if (osmoseFormat) { - Mapping_engine.setConfig(mappingConfigIRVEFromOsmose) + Mapping_engine.setConfig(config.allowed_configs.mappingConfigIRVEFromOsmose) } else { - if (engine_conf_choice !== default_engine_conf_choice && Object.keys(allowed_configs).indexOf("mappingIssy2Roues") !== -1) { - Mapping_engine.setConfig(allowed_configs[engine_conf_choice]) + if (engine_conf_choice !== default_engine_conf_choice && Object.keys(config.allowed_configs).indexOf("mappingIssy2Roues") !== -1) { + Mapping_engine.setConfig(config.allowed_configs[engine_conf_choice]) } else { - console.error('mauvais paramètre de Mapping_engine: ' + engine_conf_choice, '.\n Veuillez en sélectionner un parmi ceux autorisés avec l option --engine-config=MaConfigQuiVaBien parmi ceux ci :', Object.keys(allowed_configs)) + console.error('mauvais paramètre de Mapping_engine: ' + engine_conf_choice, '.\n Veuillez en sélectionner un parmi ceux autorisés avec l option --engine-config=MaConfigQuiVaBien parmi ceux ci :', Object.keys(config.allowed_configs)) return } } @@ -505,7 +449,7 @@ function setMappingConfigFromName(engine_conf_choice: string) { } else { console.log(' ------ on utilise mappingConfigIRVE_simple') - let mappingConfigIRVE = mappingConfigIRVE_simple + let mappingConfigIRVE = config.allowed_configs.mappingConfigIRVE_simple convertDataFromSource(sourceFilePathGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates) } @@ -515,7 +459,8 @@ function setMappingConfigFromName(engine_conf_choice: string) { * Launch conversion of dataset */ function init() { - console.log(`▗▖ ▗▖ ▗▄▖ ▗▖ ▗▄▖ ▗▖ ▗▄▖ + console.log(` +▗▖ ▗▖ ▗▄▖ ▗▖ ▗▄▖ ▗▖ ▗▄▖ ▐▌ ▐▌▐▌ ▐▌▐▌ ▐▌ ▐▌▐▌ ▐▌ ▐▌ ▐▌ ▐▌▐▌ ▐▌▐▌ ▐▌ ▐▌▐▌ ▐▌ ▐▌ ▐▙█▟▌▝▚▄▞▘▐▙▄▄▖▝▚▄▞▘▐▙▄▄▖▝▚▄▞▘ diff --git a/create_mapping.ts b/create_mapping.ts index 46c3252..9766728 100644 --- a/create_mapping.ts +++ b/create_mapping.ts @@ -84,9 +84,9 @@ export default ${configName}; `; } -// Fonction pour ajouter l'import et mettre à jour allowed_configs dans convert_to_osm_tags.ts +// Fonction pour ajouter l'import et mettre à jour allowed_configs dans config.ts function updateMainFile(configName: string): void { - const mainFilePath = 'convert_to_osm_tags.ts'; + const mainFilePath = 'config.ts'; try { let mainFileContent = fs.readFileSync(mainFilePath, 'utf8'); diff --git a/csv_to_geojson.ts b/csv_to_geojson.ts index ae95a58..3626e70 100644 --- a/csv_to_geojson.ts +++ b/csv_to_geojson.ts @@ -1,7 +1,7 @@ /** * csv_to_geojson.ts * - * Convertir un fichier CSV en GeoJSON + * Convertir un fichier CSV en GeoJSON, en précisant les colonnes latitude et longitude * * Utilisation: * @@ -10,170 +10,33 @@ */ import fs from 'fs'; import path from 'path'; -import csvParser from 'csv-parser'; import minimist from 'minimist'; -import { Feature, FeatureCollection, Point } from 'geojson'; +import { csvToGeoJSON, checkFile, countGeoJSONFeatures } from './csv_to_geojson.utils'; -interface Options { +interface CSVConversionOptions { dir: string; file: string; latColumn: string; lonColumn: string; hasHeaders: boolean; } -let counter = 0; -let counter_features = 0; -let counter_missing_lat = 0; -let counter_missing_lon = 0; -function csvToGeoJSON(options: Options): FeatureCollection { - const { dir, file, latColumn, lonColumn, hasHeaders } = options; - console.log(`options latColumn: ${latColumn}`); - console.log(`options lonColumn: ${lonColumn}`); - - const filePath = path.join(dir, file); - const features: Feature[] = []; - let geoJSON: FeatureCollection = { - type: 'FeatureCollection', - features, - }; - - let headers: string[] = []; - let headersFound = false; - - let limitOffset = 100000000; - - fs.createReadStream(filePath) - .pipe(csvParser({ headers: hasHeaders })) - .on('data', (row) => { - counter += 1; - if (!headersFound && hasHeaders) { - let keys = Object.keys(row); - keys.forEach((key) => { - headers.push(row[key]); - }); - headersFound = true; - console.log(`headers: ${headers}`); - } - if (counter > limitOffset) { - return; - } - if (headers.indexOf(latColumn) && headers.indexOf(lonColumn)) { - const lat = parseFloat(row['_' + headers.indexOf(latColumn)]); - const lon = parseFloat(row['_' + headers.indexOf(lonColumn)]); - - // Si pas d'entêtes, utiliser des noms de colonnes génériques - if (!hasHeaders) { - const properties: { [key: string]: any } = {}; - Object.keys(row).forEach((key, index) => { - properties[headers[index]] = row[key]; - }); - row = properties; - } else { - let keys = Object.keys(row); - // Utiliser les entêtes comme noms de propriétés - const properties: { [key: string]: any } = {}; - headers.forEach((header, index) => { - - properties[header] = row['_' + index]; - }); - row = properties; - } - - // filtrer la ligne du header si présente - if (lon && lat) { - if (hasHeaders && counter > 1 || !hasHeaders || counter > limitOffset) { - features.push({ - type: 'Feature', - geometry: { - type: 'Point', - coordinates: [lon, lat], - }, - properties: row, - }); - counter_features += 1; - } - } - if (headers.indexOf(latColumn) === -1) { - console.log('!!! no latColumn', row); - counter_missing_lat += 1; - } - if (headers.indexOf(lonColumn) === -1) { - console.log('!!! no lonColumn', row); - counter_missing_lon += 1; - } - } - }) - .on('end', () => { - geoJSON = { - type: 'FeatureCollection', - features, - }; - fs.writeFileSync(`${dir}/${file}.geojson`, JSON.stringify(geoJSON, null, 2)); - console.log(`GeoJSON créé avec succès : ${file}.geojson`); - console.log(`geoJSON lines: ${counter}`); - console.log(`geoJSON lines missing lat: ${counter_missing_lat}`); - console.log(`geoJSON lines missing lon: ${counter_missing_lon}`); - console.log(`geoJSON lines features: ${counter_features}`); - }); - - return geoJSON; -} - -const args = minimist(process.argv.slice(2), { +// config des arguments de conversion +const args = minimist(process.argv.slice(2), { alias: { - dir: 'd', - file: 'f', - latColumn: 'lat', - lonColumn: 'lon', - hasHeaders: 'h', + dir: 'd', // dossier source + file: 'f', // fichier source + // infos pour un fichier CSV en source au lieu d'un fichier geojson + latColumn: 'lat', // colonne latitude + lonColumn: 'lon', // colonne longitude + hasHeaders: 'h', // headers présents }, default: { hasHeaders: true, }, }); -function checkFile(args: Options) { - let filePath = path.join(args.dir, args.file); - let lineCount = 0; - - // Vérifier si le fichier existe - if (!fs.existsSync(filePath)) { - throw new Error(`Le fichier CSV ${filePath} n'existe pas`); - } else { - console.log(`Le fichier CSV ${filePath} existe`); - } - - fs.createReadStream(filePath) - .on('data', () => { - lineCount++; - }) - .on('end', () => { - console.log(`Nombre de lignes dans le fichier CSV : ${Math.floor(lineCount)}`); - }); - - -} -function countGeoJSONFeatures(args: Options) { - const filePath = path.join(args.dir, `${args.file}.geojson`); - - // Vérifier si le fichier GeoJSON existe - if (!fs.existsSync(filePath)) { - console.log(`Le fichier GeoJSON ${filePath} n'existe pas`); - return; - } - - // Lire et parser le fichier GeoJSON - const geoJSON = JSON.parse(fs.readFileSync(filePath, 'utf8')); - - // Compter le nombre de features - const featureCount = geoJSON.features?.length || 0; - - console.log(`Nombre de features dans le GeoJSON : ${featureCount}`); - return featureCount; -} checkFile(args); csvToGeoJSON(args); -// Appeler la fonction après la création du GeoJSON countGeoJSONFeatures(args); diff --git a/csv_to_geojson.utils.ts b/csv_to_geojson.utils.ts new file mode 100644 index 0000000..f7b156d --- /dev/null +++ b/csv_to_geojson.utils.ts @@ -0,0 +1,153 @@ + +import { FeatureCollection, Feature, Point } from "geojson"; +import * as fs from 'fs'; +import * as path from 'path'; +import csvParser from 'csv-parser'; + + +let counter = 0; +let counter_features = 0; +let counter_missing_lat = 0; +let counter_missing_lon = 0; + + +function csvToGeoJSON(options: Options): FeatureCollection { + const { dir, file, latColumn, lonColumn, hasHeaders } = options; + + console.log(`options latColumn: ${latColumn}`); + console.log(`options lonColumn: ${lonColumn}`); + + const filePath = path.join(dir, file); + const features: Feature[] = []; + let geoJSON: FeatureCollection = { + type: 'FeatureCollection', + features, + }; + + + let headers: string[] = []; + let headersFound = false; + let limitOffset = 100000000; + + fs.createReadStream(filePath) + .pipe(csvParser({ headers: hasHeaders })) + .on('data', (row) => { + counter += 1; + if (!headersFound && hasHeaders) { + let keys = Object.keys(row); + keys.forEach((key) => { + headers.push(row[key]); + }); + headersFound = true; + console.log(`headers: ${headers}`); + } + if (counter > limitOffset) { + return; + } + if (headers.indexOf(latColumn) && headers.indexOf(lonColumn)) { + const lat = parseFloat(row['_' + headers.indexOf(latColumn)]); + const lon = parseFloat(row['_' + headers.indexOf(lonColumn)]); + + // Si pas d'entêtes, utiliser des noms de colonnes génériques + if (!hasHeaders) { + const properties: { [key: string]: any } = {}; + Object.keys(row).forEach((key, index) => { + properties[headers[index]] = row[key]; + }); + row = properties; + } else { + let keys = Object.keys(row); + // Utiliser les entêtes comme noms de propriétés + const properties: { [key: string]: any } = {}; + headers.forEach((header, index) => { + + properties[header] = row['_' + index]; + }); + row = properties; + } + + // filtrer la ligne du header si présente + if (lon && lat) { + if (hasHeaders && counter > 1 || !hasHeaders || counter > limitOffset) { + features.push({ + type: 'Feature', + geometry: { + type: 'Point', + coordinates: [lon, lat], + }, + properties: row, + }); + counter_features += 1; + } + } + if (headers.indexOf(latColumn) === -1) { + console.log('!!! no latColumn', row); + counter_missing_lat += 1; + } + if (headers.indexOf(lonColumn) === -1) { + console.log('!!! no lonColumn', row); + counter_missing_lon += 1; + } + } + }) + .on('end', () => { + geoJSON = { + type: 'FeatureCollection', + features, + }; + fs.writeFileSync(`${dir}/${file}.geojson`, JSON.stringify(geoJSON, null, 2)); + console.log(`GeoJSON créé avec succès : ${file}.geojson`); + console.log(`geoJSON lines: ${counter}`); + console.log(`geoJSON lines missing lat: ${counter_missing_lat}`); + console.log(`geoJSON lines missing lon: ${counter_missing_lon}`); + console.log(`geoJSON lines features: ${counter_features}`); + }); + + return geoJSON; +} + +function checkFile(args: Options) { + let filePath = path.join(args.dir, args.file); + let lineCount = 0; + + // Vérifier si le fichier existe + if (!fs.existsSync(filePath)) { + throw new Error(`Le fichier CSV ${filePath} n'existe pas`); + } else { + console.log(`Le fichier CSV ${filePath} existe`); + } + + fs.createReadStream(filePath) + .on('data', () => { + lineCount++; + }) + .on('end', () => { + console.log(`Nombre de lignes dans le fichier CSV : ${Math.floor(lineCount)}`); + }); + + +} +function countGeoJSONFeatures(args: Options) { + const filePath = path.join(args.dir, `${args.file}.geojson`); + + // Vérifier si le fichier GeoJSON existe + if (!fs.existsSync(filePath)) { + console.log(`Le fichier GeoJSON ${filePath} n'existe pas`); + return; + } + + // Lire et parser le fichier GeoJSON + const geoJSON = JSON.parse(fs.readFileSync(filePath, 'utf8')); + + // Compter le nombre de features + const featureCount = geoJSON.features?.length || 0; + + console.log(`Nombre de features dans le GeoJSON : ${featureCount}`); + return featureCount; +} + +export { + csvToGeoJSON, + checkFile, + countGeoJSONFeatures +}; \ No newline at end of file diff --git a/mappings/converters/configIRVE.ts b/mappings/converters/configIRVE.ts index 371cfae..9335aef 100644 --- a/mappings/converters/configIRVE.ts +++ b/mappings/converters/configIRVE.ts @@ -19,7 +19,7 @@ const MappingIRVE: MappingConfigType = { * select only certain points from the source */ filters: { - // offset: 1, // limiter à une feature pour faire des tests + // offset: 10, // limiter à une feature pour faire des tests enable_coordinates_filter: false, enable_properties_filter: true, // filter_points_older_than_year: 2024, @@ -110,21 +110,25 @@ const MappingIRVE: MappingConfigType = { key_converted: 'socket:typee', ignore_if_falsy: true, convert_to_boolean_value: true, + keep_only_max_in_enum: true, }, prise_type_2: { key_converted: 'socket:type2', ignore_if_falsy: true, convert_to_boolean_value: true, + keep_only_max_in_enum: true, }, prise_type_combo_ccs: { key_converted: 'socket:type2_combo', ignore_if_falsy: true, convert_to_boolean_value: true, + keep_only_max_in_enum: true, }, prise_type_chademo: { key_converted: 'socket:chademo', ignore_if_falsy: true, convert_to_boolean_value: true, + keep_only_max_in_enum: true, }, // ******** champs plus complexes horaires: 'opening_hours', // déjà au bon format, enfin, en général. vérifier avec le validateur josm. diff --git a/mappings/converters/configPlanningFamilial.ts b/mappings/converters/configPlanningFamilial.ts index a558e1b..e093de4 100644 --- a/mappings/converters/configPlanningFamilial.ts +++ b/mappings/converters/configPlanningFamilial.ts @@ -1,6 +1,10 @@ /** * commerces adhérant à Ask Angela * https://wiki.openstreetmap.org/wiki/Tag:healthcare:speciality%3Dfamily_planning + * détails en Français https://wiki.openstreetmap.org/wiki/FR:Tag:healthcare:speciality%3Dfamily_planning +* +* Site officiel: + * https://www.planning-familial.org/fr */ import MappingConfigType from "../mapping-config.type"; @@ -14,7 +18,7 @@ const MappingPlanningFamlial: MappingConfigType = { // source faite à partir de data scraping du site français geojson_path: '', url: 'https://www.planning-familial.org/fr', - overpass_query:` + overpass_query: ` [out:json][timeout:200]; {{geocodeArea:"France"}}->.searchArea; nwr["healthcare:speciality"="family_planning"](area.searchArea); @@ -29,20 +33,20 @@ const MappingPlanningFamlial: MappingConfigType = { tags_to_ignore_if_value_is: ['Non renseigne'], tags: { adresse: "addr:full", - nom : { + nom: { key_converted: 'name', convert_to_name: true, }, - telephone : { + telephone: { key_converted: 'contact:phone', convert_to_phone: true, }, - "contact:website":"contact:website", - "family_planning:handles:violences" : "family_planning:handles:violences", - "family_planning:handles:sexualities" : "family_planning:handles:sexualities", - "family_planning:handles:detection" : "family_planning:handles:detection", - "family_planning:handles:abortion" : "family_planning:handles:abortion", - "family_planning:handles:contraception" : "family_planning:handles:contraception", + "contact:website": "contact:website", + "family_planning:handles:violences": "family_planning:handles:violences", + "family_planning:handles:sexualities": "family_planning:handles:sexualities", + "family_planning:handles:detection": "family_planning:handles:detection", + "family_planning:handles:abortion": "family_planning:handles:abortion", + "family_planning:handles:contraception": "family_planning:handles:contraception", // ******* opendata de toulouse END ************** } } diff --git a/mappings/engine.ts b/mappings/engine.ts index 25eae92..c03ead6 100644 --- a/mappings/engine.ts +++ b/mappings/engine.ts @@ -1,34 +1,25 @@ -import custom_utils from './utils' + import MappingConfigType from "./mapping-config.type"; import Formatters from "./formatters"; +import config from "../config"; +import custom_utils from "./utils"; +import { detectSocketOutputFromFeaturePoint } from "./irve.utils"; const { debugLog, find_max_in_string, truncate_enums_to_limit } = custom_utils -let listOfBooleanKeys = [ - "prise_type_ef", - "prise_type_2", - "prise_type_combo_ccs", - "prise_type_chademo", - "gratuit", - "paiement_acte", - "paiement_cb", - "cable_t2_attache" -] -function boolToAddable(someBooleanValue: boolean) { - return someBooleanValue ? 1 : 0 -} -export default class { +/** + * Class that helps to convert values into predefined formats + */ +export default class MappingEngine { mapping_config: any = {} public stats: any; - truthyValues = [true, 'true', 'True', 'TRUE', '1', 'yes', 1] - falsyValues = [false, 'false', 'False', 'FALSE', '0', 'no', 0] + private jardinage = false; private current_converted_geojson_point: any; private current_geojson_point: any; // currently converting point - private list_of_points: any; // list of geojson points constructor(mappingConfig: MappingConfigType) { @@ -55,12 +46,9 @@ export default class { mapFeaturePoint(featurePointGeoJson: any) { let geoJSONConvertedPoint: any = {} - - geoJSONConvertedPoint.properties = { ...this.mapping_config.default_properties_of_point } geoJSONConvertedPoint.type = featurePointGeoJson.type geoJSONConvertedPoint.geometry = featurePointGeoJson.geometry - this.current_converted_geojson_point = geoJSONConvertedPoint return geoJSONConvertedPoint @@ -73,7 +61,7 @@ export default class { */ isBooleanKey(pointKeyName: string): boolean { - return listOfBooleanKeys.indexOf(pointKeyName) !== -1 + return config.listOfBooleanKeys.indexOf(pointKeyName) !== -1 } /** @@ -205,7 +193,9 @@ export default class { debugLog('mapElementFromConf: convert', pointKeyName) debugLog('mapElementFromConf: mapping keys:', mappingKeys) - this.convertProperty(pointKeyName, mappingKeys, featurePoint, newProperties) + this.convertProperty({ + pointKeyName, mappingKeys, featurePoint, newProperties + }) }) @@ -217,13 +207,20 @@ export default class { } /** - * convertit une propriété en une autre selon la config de mapping + * convertit une propriété en une autre selon la config de mapping chargée * @param pointKeyName * @param mappingKeys * @param featurePoint * @param newProperties */ - convertProperty(pointKeyName: string, mappingKeys: any, featurePoint: any, newProperties: any) { + convertProperty(options: { + pointKeyName: string, + mappingKeys: any, + featurePoint: any, + newProperties: any + }) { + + const { pointKeyName, mappingKeys, featurePoint, newProperties } = options this.current_geojson_point = featurePoint let originalValue = '' @@ -234,7 +231,7 @@ export default class { } else { originalValue = featurePoint.properties[pointKeyName] } - let intOriginalValue = parseInt(originalValue) + let mappingValueObject: any = ''; @@ -343,12 +340,12 @@ export default class { // on met donc truthy_value: '1' debugLog('truthy_value', originalValue) - if (this.truthyValues.indexOf(originalValue) !== -1) { + if (custom_utils.truthyValues.indexOf(originalValue) !== -1) { convertedValue = configObject.truthy_value } } if (configObject.falsy_value) { - if (this.falsyValues.indexOf(originalValue) !== -1) { + if (custom_utils.falsyValues.indexOf(originalValue) !== -1) { convertedValue = configObject.falsy_value } } @@ -369,93 +366,19 @@ export default class { // avec une fonction de transformation des valeurs // parmi le domaine du jeu de données // nécessite une clé conditionnelle à la valeur true d'autres clés converties. + if (configObject.keep_only_max_in_enum) { + let max = custom_utils.find_max_in_string(originalValue) + if (max && max < 401) { + convertedValue = max + ' kW' + } else { + convertedValue = originalValue + } + } if (configObject.socket_output_find_correspondances) { - // trouver à quel socket ça correspond - // si y'a plusieurs sockets, utiliser socket:max:output - let we_use_max_output = false; - let has_prise_type_2: boolean = this.isTruthyValue(this.current_geojson_point.properties.prise_type_2) || false - let has_prise_type_combo_ccs: boolean = this.isTruthyValue(this.current_geojson_point.properties.prise_type_combo_ccs) || false - let prise_type_chademo: boolean = this.isTruthyValue(this.current_geojson_point.properties.prise_type_chademo) || false - let prise_type_ef: boolean = this.isTruthyValue(this.current_geojson_point.properties.prise_type_ef) || false - let prise_type_e: boolean = this.isTruthyValue(this.current_geojson_point.properties.prise_type_e) || false - let prise_type_autre: boolean = this.isTruthyValue(this.current_geojson_point.properties.prise_type_autre) || false - - let countOfSockets = (boolToAddable(has_prise_type_2) + boolToAddable(has_prise_type_combo_ccs) + boolToAddable(prise_type_chademo) + - boolToAddable(prise_type_ef) + boolToAddable(prise_type_autre) + boolToAddable(prise_type_e) - ); - if (countOfSockets > 0) { - we_use_max_output = true; - } - // ajouter les tags de socket newProperties - - let converted_value = find_max_in_string(originalValue.replaceAll('.00', '').replaceAll(',', '.')) - let max_output = 401 - // do not limit accepted values - let out = '' - - if (intOriginalValue < max_output) { - // rajouter l'unité de puissance kW dans la valeur - out = converted_value + ' kW' - - } else { - // prise en charge des valeurs en Watts et non en kW. - debugLog('too high kW value detected', originalValue) - if (intOriginalValue > 1000 && intOriginalValue < 401000) { - - let kilowatts = (converted_value / 1000).toFixed(2); - out = ('' + kilowatts + ' kW').replaceAll('.00', '') - debugLog('valeurs en Watts out', out, 'original:', originalValue) - this.stats.power_output++ - } - } - out = (out).replaceAll('.00', '') - - // debug land - if (has_prise_type_combo_ccs) { - newProperties['socket:type2_combo:output'] = out; - this.stats.power_output++ - } - - if (we_use_max_output) { - newProperties['charging_station:output'] = out; - } else { - if (has_prise_type_2 && prise_type_e) { - newProperties['socket:type_2:output'] = out; - this.stats.power_output++ - debugLog('2 prises, attribuer la plus haute valeur à la type 2', out) - } - - if (countOfSockets === 1) { - - if (has_prise_type_2) { - newProperties['socket:type_2:output'] = out; - newProperties['socket:type_2'] = 1; - this.stats.power_output++ - - } - if (has_prise_type_combo_ccs) { - newProperties['socket:type2_combo:output'] = out; - newProperties['socket:type2_combo'] = 1; - this.stats.power_output++ - - } - if (prise_type_chademo) { - newProperties['socket:chademo:output'] = out; - newProperties['socket:chademo'] = 1; - this.stats.power_output++ - } - if (prise_type_e) { - newProperties['socket:typee:output'] = out; - newProperties['socket:typee'] = 1; - this.stats.power_output++ - } - } else { - debugLog('no sockets', this.current_geojson_point.properties.ref) - } - } - convertedValue = out; - return out + convertedValue = detectSocketOutputFromFeaturePoint({ + pointKeyName, mappingKeys, featurePoint, newProperties, originalValue + }) } if (configObject.invert_boolean_value) { @@ -486,10 +409,10 @@ export default class { if (configObject.remove_original_key) { remove_original_key = true } - if (configObject.ignore_if_falsy && this.falsyValues.indexOf(originalValue) !== -1) { + if (configObject.ignore_if_falsy && custom_utils.falsyValues.indexOf(originalValue) !== -1) { remove_original_key = true } - if (configObject.ignore_if_truthy && this.truthyValues.indexOf(originalValue) !== -1) { + if (configObject.ignore_if_truthy && custom_utils.truthyValues.indexOf(originalValue) !== -1) { remove_original_key = true } @@ -542,12 +465,12 @@ export default class { // convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value // exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1". // on met donc truthy_value: '1' - if (this.truthyValues.indexOf(originalValue) !== -1) { + if (custom_utils.truthyValues.indexOf(originalValue) !== -1) { convertedValue = conditionnalConfig.truthy_value } } if (conditionnalConfig.falsy_value) { - if (this.falsyValues.indexOf(originalValue) !== -1) { + if (custom_utils.falsyValues.indexOf(originalValue) !== -1) { convertedValue = conditionnalConfig.falsy_value } } @@ -607,26 +530,16 @@ export default class { return newProperties; } - private isTruthyValue(someValue: string) { - let convertedValue; - if (this.truthyValues.indexOf(someValue) !== -1) { - convertedValue = true - } - if (this.falsyValues.indexOf(someValue) !== -1) { - convertedValue = false - } - return convertedValue - } private convertToYesOrNo(originalValue: any) { debugLog('convertProperty: ==========> original value', originalValue) let convertedValue = ''; - if (this.truthyValues.indexOf(originalValue) !== -1) { + if (custom_utils.truthyValues.indexOf(originalValue) !== -1) { convertedValue = 'yes' } else { debugLog('convertProperty: ==========> !!! NOT in truthy values', originalValue) } - if (this.falsyValues.indexOf(originalValue) !== -1) { + if (custom_utils.falsyValues.indexOf(originalValue) !== -1) { convertedValue = 'no' } else { debugLog('convertProperty: ==========> !!! NOT in falsy values', originalValue) @@ -637,12 +550,12 @@ export default class { private convertToBoolean(originalValue: any) { debugLog('convertProperty: ==========> original value', originalValue) let convertedValue; - if (this.truthyValues.indexOf(originalValue) !== -1) { + if (custom_utils.truthyValues.indexOf(originalValue) !== -1) { convertedValue = true } else { debugLog('convertProperty: ==========> !!! NOT in truthy values', originalValue) } - if (this.falsyValues.indexOf(originalValue) !== -1) { + if (custom_utils.falsyValues.indexOf(originalValue) !== -1) { convertedValue = false } else { debugLog('convertProperty: ==========> !!! NOT in falsy values', originalValue) diff --git a/mappings/irve.utils.ts b/mappings/irve.utils.ts new file mode 100644 index 0000000..6357f31 --- /dev/null +++ b/mappings/irve.utils.ts @@ -0,0 +1,109 @@ +import config from "../config"; +import custom_utils from "./utils"; + +const { debugLog, find_max_in_string, boolToAddable } = custom_utils + + +function detectSocketOutputFromFeaturePoint(options: { + pointKeyName: string, + mappingKeys: any, + featurePoint: any, + newProperties: any, + originalValue: string +}) { + + const { pointKeyName, mappingKeys, featurePoint, newProperties, originalValue } = options + let intOriginalValue = parseInt(originalValue) + + // trouver à quel socket ça correspond + // si y'a plusieurs sockets, utiliser socket:max:output + let we_use_max_output = false; + let has_prise_type_2: boolean = custom_utils.isTruthyValue(featurePoint.properties.prise_type_2) || false + let has_prise_type_combo_ccs: boolean = custom_utils.isTruthyValue(featurePoint.properties.prise_type_combo_ccs) || false + let prise_type_chademo: boolean = custom_utils.isTruthyValue(featurePoint.properties.prise_type_chademo) || false + let prise_type_ef: boolean = custom_utils.isTruthyValue(featurePoint.properties.prise_type_ef) || false + let prise_type_e: boolean = custom_utils.isTruthyValue(featurePoint.properties.prise_type_e) || false + let prise_type_autre: boolean = custom_utils.isTruthyValue(featurePoint.properties.prise_type_autre) || false + + let countOfSockets = (boolToAddable(has_prise_type_2) + boolToAddable(has_prise_type_combo_ccs) + boolToAddable(prise_type_chademo) + + boolToAddable(prise_type_ef) + boolToAddable(prise_type_autre) + boolToAddable(prise_type_e) + ); + if (countOfSockets > 0) { + we_use_max_output = true; + } + // ajouter les tags de socket newProperties + // certains producteurs de données donnent des énumérations de puissances + // on prend la valeur max + let converted_value = find_max_in_string(originalValue.replaceAll('.00', '').replaceAll(',', '.')) + + // do not limit accepted values + let out = '' + let out_number = 0 + + if (intOriginalValue < config.irve_max_output) { + // rajouter l'unité de puissance kW dans la valeur + out_number = converted_value + out = converted_value + ' kW' + + } else { + // prise en charge des valeurs en Watts et non en kW. + debugLog('too high kW value detected', originalValue) + if (intOriginalValue > 1000 && intOriginalValue < (config.irve_max_output * 1000)) { + + let kilowatts = (converted_value / 1000).toFixed(2); + out = ('' + kilowatts + ' kW').replaceAll('.00', '') + debugLog('valeurs en Watts out', out, 'original:', originalValue) + + } + } + out = (out).replaceAll('.0', '') + + + if (has_prise_type_combo_ccs && countOfSockets === 1) { + newProperties['socket:type2_combo:output'] = out; + + } + + if (we_use_max_output) { + newProperties['charging_station:output'] = out; + } else { + if (has_prise_type_2 && prise_type_e) { + newProperties['socket:type_2:output'] = out; + + debugLog('2 prises, attribuer la plus haute valeur à la type 2', out) + } + // deviner les décomptes de sockets + if (countOfSockets === 1) { + + if (has_prise_type_2) { + newProperties['socket:type_2:output'] = out; + newProperties['socket:type_2'] = 1; + + + } + if (has_prise_type_combo_ccs) { + newProperties['socket:type2_combo:output'] = out; + newProperties['socket:type2_combo'] = 1; + + + } + if (prise_type_chademo) { + newProperties['socket:chademo:output'] = out; + newProperties['socket:chademo'] = 1; + + } + if (prise_type_e) { + newProperties['socket:typee:output'] = out; + newProperties['socket:typee'] = 1; + + } + } else { + debugLog('no sockets') + } + } + return out +} + +export { + detectSocketOutputFromFeaturePoint +} \ No newline at end of file diff --git a/mappings/mapping-config.type.ts b/mappings/mapping-config.type.ts index f7ef160..d1e81bd 100644 --- a/mappings/mapping-config.type.ts +++ b/mappings/mapping-config.type.ts @@ -77,6 +77,7 @@ export interface FeaturePropertyMappingConfigType { truncate_enums_to_limit?: number, conditional_values?: ConditionnalValuesConfigType, transform_function?: Function, + keep_only_max_in_enum?:boolean, [key: string]: any, } diff --git a/mappings/utils.ts b/mappings/utils.ts index cc34c7f..dfb5028 100644 --- a/mappings/utils.ts +++ b/mappings/utils.ts @@ -18,6 +18,8 @@ function debugLog(...args: any[]) { } } +const truthyValues = [true, 'true', 'True', 'TRUE', '1', 'yes', 1] +const falsyValues = [false, 'false', 'False', 'FALSE', '0', 'no', 0] let listOfBooleanKeys = [ "prise_type_ef", @@ -31,6 +33,21 @@ let listOfBooleanKeys = [ ] +function boolToAddable(someBooleanValue: boolean) { + return someBooleanValue ? 1 : 0 +} + +function isTruthyValue(someValue: string) { + let convertedValue; + if (truthyValues.indexOf(someValue) !== -1) { + convertedValue = true + } + if (falsyValues.indexOf(someValue) !== -1) { + convertedValue = false + } + return convertedValue +} + /** * * @param pointKeyName @@ -121,10 +138,20 @@ function truncate_enums_to_limit(str: string, limit: number = 255) { } export default { + // debug tools debugLog, + // typing + boolToAddable, isBooleanKey, - writeFile, + isTruthyValue, + truthyValues, + falsyValues, + // research find_max_in_string, + // formatting truncate_enums_to_limit, + prefix_phone_fr_only, + // file tools + writeFile, } diff --git a/tests/main.test.js b/tests/main.test.js index 38b3b9a..e85b3ce 100644 --- a/tests/main.test.js +++ b/tests/main.test.js @@ -177,6 +177,11 @@ describe('find max in enum', () => { expect(max).toBe(10) }) + test('value has enums with kW units', () => { + let value = '12 kW;120 kW' + let max = utils.find_max_in_string(value) + expect(max).toBe(120) + }) test('value has no enums', () => { let max = utils.find_max_in_string('10') console.log('max', max)