tykayn/wololo
1
0
Fork 0
mirror of https://forge.chapril.org/tykayn/wololo synced 2025-10-09 17:02:46 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

add tests and failsafe for values true and false

Browse source
This commit is contained in:
Tykayn 2025-05-13 23:59:21 +02:00 • committed by tykayn
parent 5b0271716c
commit cb6075fc61
10 changed files with 376 additions and 230 deletions
Download patch file Download diff file Expand all files Collapse all files

35
mappings/converters/configIRVE.ts
View file

@ -29,7 +29,7 @@ const MappingIRVE: MappingConfigType = {
enable_properties_filter: true,
// filter_points_older_than_year: 2024,
// filter_points_by_year_property: 'date_mise_en_service',
// filter_points_lesser_than_NkW: 50 // ne pas sortir les points qui ont moins de ce nombre de puissance nominale
// filter_points_lesser_than_NkW: 20 // ne pas sortir les points qui ont moins de ce nombre de puissance nominale
// add only geojson points who are found having this regex in the zipcode properties
// properties: {
// consolidated_code_postal: '^[76|27]'
@ -50,7 +50,12 @@ const MappingIRVE: MappingConfigType = {
"gratuit",
"paiement_acte",
"paiement_cb",
"cable_t2_attache"
"cable_t2_attache",
"socket:typee",
"socket:type2_combo",
"socket:chademo",
"socket:type2",
"socket:type2_cable",
],
tags: {
// ******* nombres
@ -90,12 +95,12 @@ const MappingIRVE: MappingConfigType = {
paiement_acte:
{
key_converted: 'authentication:none',
convert_to_boolean_value: true, // convertit en yes ou no
convert_to_boolean_value: true,
},
reservation: {
key_converted: 'reservation',
convert_to_boolean_value: true, // convertit en yes ou no
convert_to_boolean_value: true,
},
// observations: 'note',
nom_station: 'description',
@ -116,27 +121,21 @@ const MappingIRVE: MappingConfigType = {
,
prise_type_ef: {
key_converted: 'socket:typee',
ignore_if_falsy: true,
convert_to_boolean_value: true,
keep_only_max_in_enum: true,
keep_only_truthy_yes_or_no_without_enum: true,
},
prise_type_2: {
key_converted: 'socket:type2',
ignore_if_falsy: true,
convert_to_boolean_value: true,
keep_only_max_in_enum: true,
// on convertit en yes ou no, sans enum, et on ne garde que si on obtient "yes"
keep_only_truthy_yes_or_no_without_enum: true,
},
prise_type_combo_ccs: {
key_converted: 'socket:type2_combo',
ignore_if_falsy: true,
convert_to_boolean_value: true,
keep_only_max_in_enum: true,
keep_only_truthy_yes_or_no_without_enum: true,
},
prise_type_chademo: {
key_converted: 'socket:chademo',
ignore_if_falsy: true,
convert_to_boolean_value: true,
keep_only_max_in_enum: true,
keep_only_truthy_yes_or_no_without_enum: true,
},
// ******** champs plus complexes
horaires: 'opening_hours', // déjà au bon format, enfin, en général. vérifier avec le validateur josm.
@ -149,9 +148,7 @@ const MappingIRVE: MappingConfigType = {
accessibilite_pmr: {
key_converted: "wheelchair",
conditional_values: {
"Accessibilité inconnue": {
ignore_this_data: true, // ne pas ajouter de tag si la valeur est égale à Accessibilité inconnue.
},
"Accessible mais non réservé PMR": {
value_converted: "yes"
},

127
mappings/engine.ts
View file

@ -53,15 +53,6 @@ export default class MappingEngine {
return geoJSONConvertedPoint
}
/**
* TODO convert to mapping config property to transform_truthy
* @param pointKeyName
* @returns {boolean}
*/
isBooleanKey(pointKeyName: string): boolean {
return config.listOfBooleanKeys.indexOf(pointKeyName) !== -1
}
/**
* filter: reduce number of features
@ -174,8 +165,6 @@ export default class MappingEngine {
}
// console.log('mapElementFromConf: ============= keys mappingKeys:', featurePointPropertiesKeys.length, mappingKeys)
let newProperties = { ...this.mapping_config.default_properties_of_point }
@ -185,23 +174,17 @@ export default class MappingEngine {
basePoint.geometry = featurePoint.geometry
basePoint.properties = { ...this.mapping_config.default_properties_of_point }
// apply new properties if found in mapping config
featurePointPropertiesKeys.forEach(pointKeyName => {
// if (featurePointPropertiesKeys.indexOf(pointKeyName) !== -1) {
this.convertProperty({
pointKeyName, mappingKeys, featurePoint, newProperties
})
// }
})
basePoint.properties = newProperties
// debugLog('mapElementFromConf: basePoint', basePoint)
return basePoint
}
@ -245,7 +228,7 @@ export default class MappingEngine {
let remove_original_key = false;
debugLog('tags_to_ignore_if_value_is', this.mapping_config.tags_to_ignore_if_value_is)
if (this.mapping_config.tags_to_ignore_if_value_is && this.mapping_config.tags_to_ignore_if_value_is.length && this.mapping_config.tags_to_ignore_if_value_is?.indexOf(originalValue) !== -1) {
debugLog('(x) => ignore', originalValue, ' in ', pointKeyName)
console.log('(x) => ignore', originalValue, ' in ', pointKeyName)
remove_original_key = true;
}
@ -353,13 +336,44 @@ export default class MappingEngine {
/**
* conversion booléenne
*/
if (mappingValueObject.keep_only_truthy_yes_or_no_without_enum) {
if (originalValue.indexOf(";") === -1 && custom_utils.truthyValues.indexOf(originalValue) !== -1) {
convertedValue = 'yes'
} else {
remove_original_key = true
}
}
if (mappingValueObject.convert_to_boolean_value) {
debugLog('convertProperty: is boolean_value_conversion')
// debugLog('convertProperty: is boolean_value_conversion')
convertedValue = custom_utils.convertToYesOrNo(originalValue)
// if (pointKeyName === 'prise_type_2') {
// console.log('convertProperty: is boolean_value_conversion', pointKeyName, keyConvertedFromMapping, originalValue, '=>', convertedValue)
// newProperties[keyConvertedFromMapping] = convertedValue
// }
// console.log('convertProperty: is boolean_value_conversion', pointKeyName, keyConvertedFromMapping, originalValue, '=>', convertedValue)
if (configObject.ignore_if_falsy && !custom_utils.convertToBoolean(originalValue)) {
return newProperties
}
if (configObject.ignore_if_truthy && custom_utils.convertToBoolean(originalValue)) {
return newProperties
}
} else {
debugLog('convertProperty: is NOT having boolean_value_conversion', mappingValueObject)
}
if (configObject.invert_boolean_value) {
convertedValue = !custom_utils.convertToBoolean(originalValue) ? 'yes' : 'no'
debugLog('invert boolean', convertedValue, originalValue)
}
// gestion des puissances de bornes
@ -381,13 +395,10 @@ export default class MappingEngine {
})
}
if (configObject.invert_boolean_value) {
convertedValue = !custom_utils.convertToBoolean(originalValue) ? 'yes' : 'no'
debugLog('invert boolean', convertedValue, originalValue)
}
if (configObject.remove_stars) {
// Remplace toutes les occurrences de * de manière greedy
convertedValue = originalValue.replaceAll('*', '')
convertedValue = originalValue.replace('*', '')
debugLog('remove_stars', convertedValue, originalValue)
}
@ -409,7 +420,9 @@ export default class MappingEngine {
if (configObject.remove_original_key) {
remove_original_key = true
}
if (configObject.ignore_if_falsy && !custom_utils.convertToBoolean(originalValue)) {
if (configObject.ignore_if_falsy && (false === custom_utils.convertToBoolean(originalValue))) {
remove_original_key = true
}
if (configObject.ignore_if_truthy && custom_utils.convertToBoolean(originalValue)) {
@ -423,11 +436,16 @@ export default class MappingEngine {
}
let conditionalConfig: any = ''
// console.log('/////////// configObject.ignore_if_falsy', configObject.ignore_if_falsy, "converti en booléen", custom_utils.convertToBoolean(originalValue), "remove?", remove_original_key)
/**
* config pour une clé
* nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser
*/
if (configObject.conditional_values) {
if (!remove_original_key && configObject.conditional_values) {
// convert numbers from json to string to compare them correctly
originalValue = '' + originalValue
@ -440,13 +458,14 @@ export default class MappingEngine {
// sauf si on a activé l'option allow_unspecified_conditional_values dans la MappingConfigType
if (!this.mapping_config.allow_unspecified_conditional_values && foundValue === -1) {
// console.log('(x) => ignore', originalValue, ' in ', pointKeyName)
// console.log('!!!!!!!!!! (x) => ignore', originalValue, ' in ', pointKeyName, 'on vire ', keyConvertedFromMapping)
remove_original_key = true
}
if (!remove_original_key) {
// console.log('-------- on garde la valeur', originalValue, ' dans ', pointKeyName)
if (foundValue !== -1) {
debugLog('found condition', foundValue)
@ -462,48 +481,47 @@ export default class MappingEngine {
remove_original_key = true;
}
let lowerKey = (originalValue + '').toLowerCase()
if (conditionalConfig.truthy_value) {
// convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value
// exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1".
// on met donc truthy_value: '1'
if (custom_utils.truthyValues.indexOf(originalValue) !== -1) {
if (custom_utils.truthyValues.indexOf(lowerKey) !== -1) {
convertedValue = conditionalConfig.truthy_value
}
}
if (conditionalConfig.falsy_value) {
if (custom_utils.falsyValues.indexOf(originalValue) !== -1) {
if (custom_utils.falsyValues.indexOf(lowerKey) !== -1) {
convertedValue = conditionalConfig.falsy_value
}
}
// use the value converted
else if (conditionalConfig.value_converted) {
if (conditionalConfig.value_converted) {
convertedValue = conditionalConfig.value_converted
}
}
// console.log('convertedValue =>', convertedValue)
if (conditionalConfig?.tags_to_add) {
debugLog('on ajoute des tags', conditionalConfig.tags_to_add)
// on peut définir un ensemble de tags à rajouter
let tagKeys = Object.keys(conditionalConfig.tags_to_add)
debugLog('conditionnalConfig.tags_to_add', conditionalConfig.tags_to_add)
tagKeys.forEach((index: any) => {
debugLog('key', index)
debugLog('value', conditionalConfig.tags_to_add[index])
newProperties[index] = conditionalConfig.tags_to_add[index]
})
}
}
}
// console.log('convertedValue =>', convertedValue)
if (conditionalConfig?.tags_to_add) {
debugLog('on ajoute des tags', conditionalConfig.tags_to_add)
// on peut définir un ensemble de tags à rajouter
let tagKeys = Object.keys(conditionalConfig.tags_to_add)
debugLog('conditionnalConfig.tags_to_add', conditionalConfig.tags_to_add)
tagKeys.forEach((index: any) => {
debugLog('key', index)
debugLog('value', conditionalConfig.tags_to_add[index])
newProperties[index] = conditionalConfig.tags_to_add[index]
})
}
} else {
debugLog('no conditional values', configObject)
}
// console.log('conditionnalConfig', conditionnalConfig, convertedValue)
debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}')
debugLog(' =============== remove_original_key', newKey, remove_original_key)
@ -514,21 +532,32 @@ export default class MappingEngine {
keysOfConfigObject = Object.keys(configObject)
debugLog('keysOfConfigObject', keysOfConfigObject)
hasKeyIgnoreThisData = (keysOfConfigObject.indexOf('ignore_this_data') !== -1)
// console.log('-------- hasKeyIgnoreThisData', hasKeyIgnoreThisData)
}
debugLog('remove_original_key && newKey && convertedValue && hasKeyIgnoreThisData', remove_original_key, newKey, convertedValue, hasKeyIgnoreThisData)
// console.log('newKey && convertedValue && !hasKeyIgnoreThisData', newKey && convertedValue && !hasKeyIgnoreThisData, newKey, convertedValue, !hasKeyIgnoreThisData)
if (remove_original_key) {
delete newProperties[pointKeyName];
// failsafe for boolean values, OSM never returns true or false
if (convertedValue == 'true') {
convertedValue = 'yes'
}
if (newKey && convertedValue && !hasKeyIgnoreThisData
if (convertedValue == 'false') {
convertedValue = 'no'
}
if (newKey && convertedValue && !hasKeyIgnoreThisData && !remove_original_key
) {
debugLog('convertedValue', convertedValue)
debugLog('convertProperty: added', newKey, (`${convertedValue}`).trim())
newProperties[newKey] = (`${convertedValue}`).trim()
}
if (remove_original_key) {
// console.log('remove_original_key', pointKeyName, originalValue, '=>', convertedValue)
delete newProperties[pointKeyName];
}
}
else {
@ -539,8 +568,6 @@ export default class MappingEngine {
}
}
// console.log('pointKeyName', pointKeyName)
return newProperties;
}

1
mappings/mapping-config.type.ts
View file

@ -80,6 +80,7 @@ export interface FeaturePropertyMappingConfigType {
conditional_values?: ConditionnalValuesConfigType,
transform_function?: Function,
keep_only_max_in_enum?: boolean,
keep_only_truthy_yes_or_no_without_enum?: boolean,
[key: string]: any,
}

58
mappings/utils.ts
View file

@ -1,4 +1,5 @@
import * as fs from 'node:fs'
import fetch from 'node-fetch';
let show_debug = 0
// show_debug = 1
@ -18,8 +19,8 @@ function debugLog(...args: any[]) {
}
}
const truthyValues = [true, 'true', 'True', 'TRUE', '1', 'yes', 1]
const falsyValues = [false, 'false', 'False', 'FALSE', '0', 'no', 0]
const truthyValues = [true, 'true', '1', 'yes', 1]
const falsyValues = [false, 'false', '0', 'no', 0]
let listOfBooleanKeys = [
"prise_type_ef",
@ -39,10 +40,10 @@ function boolToAddable(someBooleanValue: boolean) {
function isTruthyValue(someValue: string) {
let convertedValue;
if (truthyValues.indexOf(someValue) !== -1) {
if (truthyValues.indexOf(someValue + ''.toLowerCase()) !== -1) {
convertedValue = true
}
if (falsyValues.indexOf(someValue) !== -1) {
if (falsyValues.indexOf(someValue + ''.toLowerCase()) !== -1) {
convertedValue = false
}
return convertedValue
@ -138,39 +139,35 @@ function truncate_enums_to_limit(str: string, limit: number = 255) {
}
function convertToBoolean(originalValue: any): boolean {
if (truthyValues.indexOf(originalValue) !== -1) {
if (truthyValues.indexOf(originalValue + ''.toLowerCase()) !== -1) {
return true;
}
if (falsyValues.indexOf(originalValue) !== -1) {
if (falsyValues.indexOf(originalValue + ''.toLowerCase()) !== -1) {
return false;
}
return false; // valeur par défaut
}
function convertToYesOrNo(originalValue: any) {
function convertToYesOrNo(originalValue: any): string {
let intermediateValue = '' + originalValue
let isEnumeration = false;
let convertedValue = '';
// handle lists with ; as separator
if (intermediateValue.includes(';')) {
isEnumeration = true;
intermediateValue = intermediateValue.split(';')[0]
}
// on ne peut pas conclure ce que l'on doit garder avec une liste
if (isEnumeration) {
if (intermediateValue.indexOf(';') !== -1) {
return ''
}
intermediateValue = intermediateValue.split(';')[0]
debugLog('convertProperty: ==========> original value', originalValue, intermediateValue)
if (truthyValues.indexOf(originalValue) !== -1) {
convertedValue = 'yes'
if (truthyValues.indexOf((originalValue + '').toLowerCase()) !== -1) {
return 'yes'
} else {
debugLog('convertProperty: ==========> !!! NOT in truthy values', originalValue)
}
if (falsyValues.indexOf(originalValue) !== -1) {
convertedValue = 'no'
if (falsyValues.indexOf((originalValue + '').toLowerCase()) !== -1) {
return 'no'
} else {
debugLog('convertProperty: ==========> !!! NOT in falsy values', originalValue)
}
@ -178,6 +175,30 @@ function convertToYesOrNo(originalValue: any) {
return convertedValue;
}
async function replaceFile(sourceFilePathGeoJson: string, url: string) {
const response = await fetch(url)
if (!response.ok) {
throw new Error(`Erreur lors du téléchargement: ${response.status} ${response.statusText}`)
}
// afficher la taille du fichier téléchargé
const contentLength = response.headers.get('content-length')
const data = await response.text()
if (contentLength) {
const sizeInMB = (parseInt(contentLength) / (1024 * 1024)).toFixed(2)
console.log(`Taille du fichier téléchargé: ${sizeInMB} Mo`)
} else {
// mesurer la taille des données
const sizeInMB = (data.length / (1024 * 1024)).toFixed(2)
console.log(`Taille des données: ${sizeInMB} Mo`)
}
fs.writeFileSync(sourceFilePathGeoJson, data)
console.log('fichier téléchargé avec succès:', sourceFilePathGeoJson)
}
export default {
// debug tools
debugLog,
@ -198,4 +219,5 @@ export default {
prefix_phone_fr_only,
// file tools
writeFile,
replaceFile,
}