import MappingConfigType from "./mapping-config.type"; import Formatters from "./formatters"; import config from "../config"; import custom_utils from "./utils"; import { detectSocketOutputFromFeaturePoint } from "./irve.utils"; const { debugLog, find_max_in_string, truncate_enums_to_limit } = custom_utils /** * Class that helps to convert values into predefined formats */ export default class MappingEngine { mapping_config: any = {} public stats: any; private jardinage = false; private current_converted_geojson_point: any; private current_geojson_point: any; // currently converting point constructor(mappingConfig: MappingConfigType) { this.setConfig(mappingConfig) this.stats = { filtered_by_excluded_tags: 0, phones_updated: 0, power_output: 0, phones_updated_list: [], phones_not_updated: 0 } } setConfig(mappingConfig: MappingConfigType) { debugLog('load config', mappingConfig.config_name) debugLog('load config', mappingConfig.config_name) this.mapping_config = mappingConfig } getConfig() { return this.mapping_config } mapFeaturePoint(featurePointGeoJson: any) { let geoJSONConvertedPoint: any = {} geoJSONConvertedPoint.properties = { ...this.mapping_config.default_properties_of_point } geoJSONConvertedPoint.type = featurePointGeoJson.type geoJSONConvertedPoint.geometry = featurePointGeoJson.geometry this.current_converted_geojson_point = geoJSONConvertedPoint return geoJSONConvertedPoint } /** * TODO convert to mapping config property to transform_truthy * @param pointKeyName * @returns {boolean} */ isBooleanKey(pointKeyName: string): boolean { return config.listOfBooleanKeys.indexOf(pointKeyName) !== -1 } /** * filter: reduce number of features * @param offsetCount * @param listOfFeatures */ filterFeaturesByOffset(offsetCount: number, listOfFeatures: any): Array { let filteredList = listOfFeatures // TODO return filteredList } /** * filterFeaturesByPropertyRegex * TODO * @param propertyName * @param criteriaRegex * @param listOfFeatures */ filterFeaturesByPropertyRegex(propertyName: string, criteriaRegex: any, listOfFeatures: any) { let filteredList = listOfFeatures.filter((feature: any) => { return criteriaRegex.test(feature?.properties[propertyName]) }) return filteredList } /** * filter a list of geojson points if one of the given exludedKeys is present in their properties. * Example, we do not want to convert already present OSM point which have an osm_id value in their properties. * @param list * @param excludedKeys */ filterListOfPointsByExcludingIfKeyFilled(list: any, excludedKeys: Array): any[] { let newList: Array = [] list.forEach((geojsonPoint: any) => { let pointProperties = Object.keys(geojsonPoint.properties) let addPoint = true; excludedKeys.forEach((key: any) => { debugLog(key, 'pointProperties[key]', pointProperties[key]) let foundProperty: string = pointProperties[key] if (foundProperty && foundProperty !== 'null') { addPoint = false } }) if (addPoint) { // only add points that pass the not null filter newList.push(geojsonPoint) } else { this.stats.filtered_by_excluded_tags++ } }) return newList; } filterListOfPointsByExcludingIfColumnBeforeSomeYear(year: number, column: string, list_of_points: any[]): any[] { let newList: any[] = [] list_of_points.forEach((geojsonPoint: any) => { let pointProperties = Object.keys(geojsonPoint.properties) // trouver la valeur // on inclut les points dont la date de création est de l'année demandée ou supérieure if (pointProperties.includes(column) && (geojsonPoint.properties[column].substring(0, 4) * 1) >= year ) { newList.push(geojsonPoint) } }) return newList; } filterListOfPointsByExcludingIfMaxPowerIsLesserThan(minValue: number, list_of_points: any[]): any[] { let newList: any[] = [] list_of_points.forEach((geojsonPoint: any) => { let pointProperties = Object.keys(geojsonPoint.properties) // trouver la valeur // socket_output_find_correspondances if (pointProperties.includes('puissance_nominale') && 1 * (geojsonPoint.properties['puissance_nominale'].replaceAll(' kW', '')) > minValue ) { let max_power = find_max_in_string(geojsonPoint.properties['puissance_nominale']) if (max_power >= minValue) { newList.push(geojsonPoint) } } }) return newList; } /** * retuns the converted element from mapping config if present, null otherwise */ mapElementFromConf(featurePoint: any): any { debugLog('mapElementFromConf: mapElementFromConf', featurePoint) if (!this.mapping_config) { throw new Error('no config was loaded in the mapping engine. use setConfig(my_mapping_config) on this instance of mapping engine before using this. Your config should be typed to MappingConfigType Type.') } debugLog('mapElementFromConf: config_name', this.mapping_config.config_name) let mappingKeys = Object.keys(this.mapping_config.tags) let featurePointPropertiesKeys: string[] = []; if (this.mapping_config.osmose) { featurePointPropertiesKeys = Object.keys(featurePoint.properties.fixes[0][0].create); } else { featurePointPropertiesKeys = Object.keys(featurePoint.properties); } // console.log('mapElementFromConf: ============= keys mappingKeys:', featurePointPropertiesKeys.length, mappingKeys) let newProperties = { ...this.mapping_config.default_properties_of_point } // reinit properties of current point let basePoint = Object.create(featurePoint) basePoint.type = featurePoint.type basePoint.geometry = featurePoint.geometry basePoint.properties = { ...this.mapping_config.default_properties_of_point } // apply new properties if found in mapping config featurePointPropertiesKeys.forEach(pointKeyName => { // if (featurePointPropertiesKeys.indexOf(pointKeyName) !== -1) { this.convertProperty({ pointKeyName, mappingKeys, featurePoint, newProperties }) // } }) basePoint.properties = newProperties // debugLog('mapElementFromConf: basePoint', basePoint) return basePoint } /** * convertit une propriété en une autre selon la config de mapping chargée * @param pointKeyName * @param mappingKeys * @param featurePoint * @param newProperties */ convertProperty(options: { pointKeyName: string, mappingKeys: any, featurePoint: any, newProperties: any }) { const { pointKeyName, mappingKeys, featurePoint, newProperties } = options this.current_geojson_point = featurePoint let originalValue = '' if (this.mapping_config.osmose) { originalValue = featurePoint.properties.fixes[0][0].create[pointKeyName] } else { originalValue = featurePoint.properties[pointKeyName] } let mappingValueObject: any = ''; if (mappingKeys.indexOf(pointKeyName) !== -1) { mappingValueObject = this.mapping_config.tags[pointKeyName] debugLog('convertProperty: mappingValueObject ', mappingValueObject) } debugLog(' ------ convertProperty: pointKeyName', pointKeyName) // debugLog('convertProperty: mappingKeys', mappingKeys) let remove_original_key = false; debugLog('tags_to_ignore_if_value_is', this.mapping_config.tags_to_ignore_if_value_is) if (this.mapping_config.tags_to_ignore_if_value_is && this.mapping_config.tags_to_ignore_if_value_is.length && this.mapping_config.tags_to_ignore_if_value_is?.indexOf(originalValue) !== -1) { debugLog('(x) => ignore', originalValue, ' in ', pointKeyName) remove_original_key = true; } if (this.jardinage) { debugLog(' ------ on fait du jardinage') debugLog(' ------ mode mise en qualité activé') debugLog(' ------ les données en entrée sont des infos geojson extraites depuis overpass turbo.') debugLog(' ------ les clés des objets sont donc déjà dans le format de tag OSM,' + 'ne pas les convertir pour les mettre en qualité selon le modèle de mapping.') } if (this.mapping_config.add_not_mapped_tags_too && (mappingKeys.indexOf(pointKeyName) === -1)) { /** * add all unmapped tags is enabled */ debugLog(' ------ add all unmapped tags is enabled') newProperties[pointKeyName] = originalValue; } else { /** * only use existing keys */ // console.log("only use existing keys,", pointKeyName) if (mappingKeys.indexOf(pointKeyName) !== -1) { let valueConvertedFromMapping = featurePoint.properties[pointKeyName] let keyConvertedFromMapping = mappingKeys[mappingKeys.indexOf(pointKeyName)] let mappingConfigOfTag = this.mapping_config.tags[pointKeyName] debugLog('========== mappingConfigOfTag', mappingConfigOfTag) debugLog('convertProperty: found element', pointKeyName, '=>', keyConvertedFromMapping, 'value : ', valueConvertedFromMapping) let convertedValue = originalValue let typeOfConfigForKey = typeof mappingConfigOfTag // console.log('typeOfConfigForKey', typeOfConfigForKey) let isStringValue = typeOfConfigForKey === 'string' let isConfigMappingObject = typeOfConfigForKey === 'object' debugLog('convertProperty: - typeofValue', typeOfConfigForKey) debugLog('convertProperty: - pointKeyName', pointKeyName) debugLog('convertProperty: - valueConvertedFromMapping', valueConvertedFromMapping) debugLog('typeof valueConvertedFromMapping === \'string\'', typeOfConfigForKey) debugLog('convertProperty: isStringValue?', valueConvertedFromMapping, isStringValue) debugLog('convertProperty: isStringValue?', valueConvertedFromMapping, isStringValue) debugLog('mappingConfigOfTag', mappingConfigOfTag) debugLog('typeOfConfigForKey', typeOfConfigForKey) /** * conversion si la clé à une config d'une string, on ne change que la clé, pas la valeur */ if (isStringValue) { debugLog('convertProperty: -- string value') debugLog('convertProperty: -- string value') debugLog('convertProperty: -- simple conversion : ', pointKeyName, '=> ', mappingConfigOfTag, '_', originalValue, '=>', valueConvertedFromMapping) debugLog('convertProperty: -- convertedValue', convertedValue) convertedValue = valueConvertedFromMapping if (convertedValue) { newProperties[mappingConfigOfTag] = convertedValue } } else { debugLog('convertProperty: no string value') } let configObject = mappingConfigOfTag if (isConfigMappingObject) { // console.log('convertProperty: is config object', configObject) let newKey: any = '' + pointKeyName if (configObject.key_converted) { newKey = configObject.key_converted debugLog('key_converted newKey', newKey) } if (configObject.transform_function) { convertedValue = configObject.transform_function(originalValue) } if (configObject.truthy_value) { // convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value // exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1". // on met donc truthy_value: '1' debugLog('truthy_value', originalValue) if (custom_utils.truthyValues.indexOf(originalValue) !== -1) { convertedValue = configObject.truthy_value } } if (configObject.falsy_value) { if (custom_utils.falsyValues.indexOf(originalValue) !== -1) { convertedValue = configObject.falsy_value } } /** * conversion booléenne */ if (mappingValueObject.convert_to_boolean_value) { debugLog('convertProperty: is boolean_value_conversion') convertedValue = this.convertToYesOrNo(originalValue) } else { debugLog('convertProperty: is NOT having boolean_value_conversion', mappingValueObject) } // gestion des puissances de bornes // avec une fonction de transformation des valeurs // parmi le domaine du jeu de données // nécessite une clé conditionnelle à la valeur true d'autres clés converties. if (configObject.keep_only_max_in_enum) { let max = custom_utils.find_max_in_string(originalValue) if (max && max < 401) { convertedValue = max + ' kW' } else { convertedValue = originalValue } } if (configObject.socket_output_find_correspondances) { convertedValue = detectSocketOutputFromFeaturePoint({ pointKeyName, mappingKeys, featurePoint, newProperties, originalValue }) } if (configObject.invert_boolean_value) { convertedValue = !this.convertToBoolean(originalValue) ? 'yes' : 'no' debugLog('invert boolean', convertedValue, originalValue) } if (configObject.remove_stars) { // Remplace toutes les occurrences de * de manière greedy convertedValue = originalValue.replaceAll('*', '') debugLog('remove_stars', convertedValue, originalValue) } if (configObject.convert_to_phone) { convertedValue = Formatters.convertToPhone(originalValue) if (originalValue !== convertedValue) { this.stats.phones_updated++ this.stats.phones_updated_list.push(convertedValue) } else { this.stats.phones_not_updated++ } debugLog('convertedValue convert_to_phone', originalValue, '=>', convertedValue) } if (configObject.convert_to_name) { convertedValue = Formatters.convertToName(originalValue) } if (configObject.remove_original_key) { remove_original_key = true } if (configObject.ignore_if_falsy && custom_utils.falsyValues.indexOf(originalValue) !== -1) { remove_original_key = true } if (configObject.ignore_if_truthy && custom_utils.truthyValues.indexOf(originalValue) !== -1) { remove_original_key = true } if (configObject.truncate_enums_to_limit) { // console.log('configObject.truncate_enums_to_limit', configObject) convertedValue = custom_utils.truncate_enums_to_limit(convertedValue, configObject.truncate_enums_to_limit) debugLog('truncate_enums_to_limit => ', convertedValue) } let conditionnalConfig: any = '' /** * config pour une clé * nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser */ if (configObject.conditional_values) { // console.log('configObject.conditional_values', configObject.conditional_values) // convert numbers from json to string to compare them correctly originalValue = '' + originalValue let keysConditionnalValues: any = Object.keys(configObject.conditional_values) let isFoundValue = keysConditionnalValues.indexOf(originalValue) conditionnalConfig = configObject.conditional_values[keysConditionnalValues[isFoundValue]] if (!remove_original_key) { if (isFoundValue !== -1) { debugLog('found condition', isFoundValue) /** ---------------------- * gestion des valeurs conditionnelles * ---------------------- */ debugLog('conditionnalConfig', conditionnalConfig) if (conditionnalConfig.ignore_this_data) { debugLog(`on ignore cette clé car sa valeur "${originalValue}" est à exclure: `, pointKeyName, '=>', newKey) remove_original_key = true; } if (conditionnalConfig.truthy_value) { // convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value // exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1". // on met donc truthy_value: '1' if (custom_utils.truthyValues.indexOf(originalValue) !== -1) { convertedValue = conditionnalConfig.truthy_value } } if (conditionnalConfig.falsy_value) { if (custom_utils.falsyValues.indexOf(originalValue) !== -1) { convertedValue = conditionnalConfig.falsy_value } } // use the value converted else if (conditionnalConfig.value_converted) { convertedValue = conditionnalConfig.value_converted } } } // console.log('convertedValue =>', convertedValue) if (conditionnalConfig?.tags_to_add) { debugLog('on ajoute des tags', conditionnalConfig.tags_to_add) // on peut définir un ensemble de tags à rajouter let tagKeys = Object.keys(conditionnalConfig.tags_to_add) debugLog('conditionnalConfig.tags_to_add', conditionnalConfig.tags_to_add) tagKeys.forEach((index: any) => { debugLog('key', index) debugLog('value', conditionnalConfig.tags_to_add[index]) newProperties[index] = conditionnalConfig.tags_to_add[index] }) } } else { debugLog('no conditional values', configObject) } // console.log('conditionnalConfig', conditionnalConfig, convertedValue) debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}') debugLog(' =============== remove_original_key', newKey, remove_original_key) let keysOfConfigObject: string[] = []; let hasKeyIgnoreThisData = false; if (configObject) { keysOfConfigObject = Object.keys(configObject) debugLog('keysOfConfigObject', keysOfConfigObject) hasKeyIgnoreThisData = (keysOfConfigObject.indexOf('ignore_this_data') !== -1) } debugLog('remove_original_key && newKey && convertedValue && hasKeyIgnoreThisData', remove_original_key, newKey, convertedValue, hasKeyIgnoreThisData) // console.log('newKey && convertedValue && !hasKeyIgnoreThisData', newKey && convertedValue && !hasKeyIgnoreThisData, newKey, convertedValue, !hasKeyIgnoreThisData) if (remove_original_key) { delete newProperties[pointKeyName]; } if (newKey && convertedValue && !hasKeyIgnoreThisData ) { debugLog('convertedValue', convertedValue) debugLog('convertProperty: added', newKey, (`${convertedValue}`).trim()) newProperties[newKey] = (`${convertedValue}`).trim() } } else { debugLog('!!!!!! not isConfigMappingObject: ', isConfigMappingObject) } } else { debugLog('!!!!!! property not found in mappingKeys: ', pointKeyName) } } // console.log('pointKeyName', pointKeyName) return newProperties; } private convertToYesOrNo(originalValue: any) { debugLog('convertProperty: ==========> original value', originalValue) let convertedValue = ''; if (custom_utils.truthyValues.indexOf(originalValue) !== -1) { convertedValue = 'yes' } else { debugLog('convertProperty: ==========> !!! NOT in truthy values', originalValue) } if (custom_utils.falsyValues.indexOf(originalValue) !== -1) { convertedValue = 'no' } else { debugLog('convertProperty: ==========> !!! NOT in falsy values', originalValue) } return convertedValue; } private convertToBoolean(originalValue: any) { debugLog('convertProperty: ==========> original value', originalValue) let convertedValue; if (custom_utils.truthyValues.indexOf(originalValue) !== -1) { convertedValue = true } else { debugLog('convertProperty: ==========> !!! NOT in truthy values', originalValue) } if (custom_utils.falsyValues.indexOf(originalValue) !== -1) { convertedValue = false } else { debugLog('convertProperty: ==========> !!! NOT in falsy values', originalValue) } return convertedValue; } }