mirror of
https://forge.chapril.org/tykayn/wololo
synced 2025-06-20 01:34:42 +02:00
577 lines
25 KiB
TypeScript
577 lines
25 KiB
TypeScript
import MappingConfigType from "./mapping-config.type";
|
|
import Formatters from "./formatters";
|
|
import config from "../config";
|
|
import custom_utils from "./utils";
|
|
import { detectSocketOutputFromFeaturePoint } from "./irve.utils";
|
|
|
|
const { debugLog, find_max_in_string, truncate_enums_to_limit } = custom_utils
|
|
|
|
|
|
|
|
|
|
/**
|
|
* Class that helps to convert values into predefined formats
|
|
*/
|
|
export default class MappingEngine {
|
|
mapping_config: any = {}
|
|
public stats: any;
|
|
|
|
private jardinage = false;
|
|
private current_converted_geojson_point: any;
|
|
private current_geojson_point: any; // currently converting point
|
|
|
|
constructor(mappingConfig: MappingConfigType) {
|
|
this.mapping_config = mappingConfig;
|
|
|
|
this.stats = {
|
|
filtered_by_excluded_tags: 0,
|
|
phones_updated: 0,
|
|
power_output: 0,
|
|
phones_updated_list: [],
|
|
phones_not_updated: 0,
|
|
enumeration_detected: 0
|
|
}
|
|
}
|
|
|
|
setConfig(mappingConfig: MappingConfigType) {
|
|
//debugLog('load config', mappingConfig.config_name);
|
|
this.mapping_config = mappingConfig;
|
|
}
|
|
|
|
getConfig() {
|
|
return this.mapping_config
|
|
}
|
|
|
|
mapFeaturePoint(featurePointGeoJson: any) {
|
|
|
|
let geoJSONConvertedPoint: any = {}
|
|
geoJSONConvertedPoint.properties = { ...this.mapping_config.default_properties_of_point }
|
|
geoJSONConvertedPoint.type = featurePointGeoJson.type
|
|
geoJSONConvertedPoint.geometry = featurePointGeoJson.geometry
|
|
this.current_converted_geojson_point = geoJSONConvertedPoint
|
|
|
|
return geoJSONConvertedPoint
|
|
}
|
|
|
|
|
|
/**
|
|
* filter: reduce number of features
|
|
* @param offsetCount
|
|
* @param listOfFeatures
|
|
*/
|
|
filterFeaturesByOffset(offsetCount: number, listOfFeatures: any): Array<any> {
|
|
let filteredList = listOfFeatures
|
|
// TODO
|
|
return filteredList
|
|
}
|
|
|
|
/**
|
|
* filterFeaturesByPropertyRegex
|
|
* TODO
|
|
* @param propertyName
|
|
* @param criteriaRegex
|
|
* @param listOfFeatures
|
|
*/
|
|
filterFeaturesByPropertyRegex(propertyName: string, criteriaRegex: any, listOfFeatures: any) {
|
|
let filteredList = listOfFeatures.filter((feature: any) => {
|
|
return criteriaRegex.test(feature?.properties[propertyName])
|
|
})
|
|
return filteredList
|
|
}
|
|
|
|
/**
|
|
* filter a list of geojson points if one of the given exludedKeys is present in their properties.
|
|
* Example, we do not want to convert already present OSM point which have an osm_id value in their properties.
|
|
* @param list
|
|
* @param excludedKeys
|
|
*/
|
|
filterListOfPointsByExcludingIfKeyFilled(list: any, excludedKeys: Array<string>): any[] {
|
|
let newList: Array<any> = []
|
|
list.forEach((geojsonPoint: any) => {
|
|
let pointProperties = Object.keys(geojsonPoint.properties)
|
|
let addPoint = true;
|
|
excludedKeys.forEach((key: any) => {
|
|
debugLog(key, 'pointProperties[key]', pointProperties[key])
|
|
let foundProperty: string = pointProperties[key]
|
|
if (foundProperty && foundProperty !== 'null') {
|
|
addPoint = false
|
|
}
|
|
})
|
|
if (addPoint) {
|
|
// only add points that pass the not null filter
|
|
newList.push(geojsonPoint)
|
|
} else {
|
|
this.stats.filtered_by_excluded_tags++
|
|
}
|
|
})
|
|
|
|
return newList;
|
|
}
|
|
|
|
filterListOfPointsByExcludingIfColumnBeforeSomeYear(year: number, column: string, list_of_points: any[]): any[] {
|
|
let newList: any[] = []
|
|
list_of_points.forEach((geojsonPoint: any) => {
|
|
let pointProperties = Object.keys(geojsonPoint.properties)
|
|
// trouver la valeur
|
|
|
|
// on inclut les points dont la date de création est de l'année demandée ou supérieure
|
|
if (pointProperties.includes(column) &&
|
|
(geojsonPoint.properties[column].substring(0, 4) * 1) >= year
|
|
) {
|
|
newList.push(geojsonPoint)
|
|
}
|
|
})
|
|
|
|
return newList;
|
|
|
|
}
|
|
|
|
filterListOfPointsByExcludingIfMaxPowerIsLesserThan(minValue: number, list_of_points: any[]): any[] {
|
|
let newList: any[] = []
|
|
list_of_points.forEach((geojsonPoint: any) => {
|
|
let pointProperties = Object.keys(geojsonPoint.properties)
|
|
// trouver la valeur
|
|
// socket_output_find_correspondances
|
|
if (pointProperties.includes('puissance_nominale') &&
|
|
1 * (geojsonPoint.properties['puissance_nominale'].replaceAll(' kW', '')) > minValue
|
|
) {
|
|
let max_power = find_max_in_string(geojsonPoint.properties['puissance_nominale'])
|
|
if (max_power >= minValue) {
|
|
newList.push(geojsonPoint)
|
|
}
|
|
}
|
|
})
|
|
|
|
return newList;
|
|
|
|
}
|
|
|
|
/**
|
|
* retuns the converted element from mapping config if present, null otherwise
|
|
*/
|
|
mapElementFromConf(featurePoint: any): any {
|
|
debugLog('mapElementFromConf: mapElementFromConf', featurePoint)
|
|
if (!this.mapping_config) {
|
|
throw new Error('no config was loaded in the mapping engine. use setConfig(my_mapping_config) on this instance of mapping engine before using this. Your config should be typed to MappingConfigType Type.')
|
|
}
|
|
|
|
debugLog('mapElementFromConf: config_name', this.mapping_config.config_name)
|
|
let mappingKeys = Object.keys(this.mapping_config.tags)
|
|
let featurePointPropertiesKeys: string[] = [];
|
|
if (this.mapping_config.osmose) {
|
|
featurePointPropertiesKeys = Object.keys(featurePoint.properties.fixes[0][0].create);
|
|
} else {
|
|
featurePointPropertiesKeys = Object.keys(featurePoint.properties);
|
|
}
|
|
|
|
|
|
let newProperties = { ...this.mapping_config.default_properties_of_point }
|
|
|
|
|
|
// reinit properties of current point
|
|
let basePoint = Object.create(featurePoint)
|
|
basePoint.type = featurePoint.type
|
|
basePoint.geometry = featurePoint.geometry
|
|
basePoint.properties = { ...this.mapping_config.default_properties_of_point }
|
|
|
|
// apply new properties if found in mapping config
|
|
featurePointPropertiesKeys.forEach(pointKeyName => {
|
|
|
|
this.convertProperty({
|
|
pointKeyName, mappingKeys, featurePoint, newProperties
|
|
})
|
|
|
|
|
|
})
|
|
|
|
basePoint.properties = newProperties
|
|
return basePoint
|
|
}
|
|
|
|
/**
|
|
* convertit une propriété en une autre selon la config de mapping chargée
|
|
* @param pointKeyName
|
|
* @param mappingKeys
|
|
* @param featurePoint
|
|
* @param newProperties
|
|
*/
|
|
convertProperty(options: {
|
|
pointKeyName: string,
|
|
mappingKeys: any,
|
|
featurePoint: any,
|
|
newProperties: any
|
|
}) {
|
|
|
|
const { pointKeyName, mappingKeys, featurePoint, newProperties } = options
|
|
this.current_geojson_point = featurePoint
|
|
|
|
let originalValue = ''
|
|
|
|
|
|
if (this.mapping_config.osmose) {
|
|
originalValue = featurePoint.properties.fixes[0][0].create[pointKeyName]
|
|
} else {
|
|
originalValue = featurePoint.properties[pointKeyName]
|
|
}
|
|
|
|
|
|
let mappingValueObject: any = '';
|
|
|
|
if (mappingKeys.indexOf(pointKeyName) !== -1) {
|
|
mappingValueObject = this.mapping_config.tags[pointKeyName]
|
|
debugLog('convertProperty: mappingValueObject ', mappingValueObject)
|
|
}
|
|
|
|
debugLog(' ------ convertProperty: pointKeyName', pointKeyName)
|
|
// debugLog('convertProperty: mappingKeys', mappingKeys)
|
|
|
|
let remove_original_key = false;
|
|
debugLog('tags_to_ignore_if_value_is', this.mapping_config.tags_to_ignore_if_value_is)
|
|
if (this.mapping_config.tags_to_ignore_if_value_is && this.mapping_config.tags_to_ignore_if_value_is.length && this.mapping_config.tags_to_ignore_if_value_is?.indexOf(originalValue) !== -1) {
|
|
console.log('(x) => ignore', originalValue, ' in ', pointKeyName)
|
|
remove_original_key = true;
|
|
|
|
}
|
|
|
|
|
|
if (this.jardinage) {
|
|
debugLog(' ------ on fait du jardinage')
|
|
debugLog(' ------ mode mise en qualité activé')
|
|
debugLog(' ------ les données en entrée sont des infos geojson extraites depuis overpass turbo.')
|
|
debugLog(' ------ les clés des objets sont donc déjà dans le format de tag OSM,' +
|
|
'ne pas les convertir pour les mettre en qualité selon le modèle de mapping.')
|
|
}
|
|
if (this.mapping_config.add_not_mapped_tags_too && (mappingKeys.indexOf(pointKeyName) === -1)) {
|
|
/**
|
|
* add all unmapped tags is enabled
|
|
*/
|
|
debugLog(' ------ add all unmapped tags is enabled')
|
|
newProperties[pointKeyName] = originalValue;
|
|
|
|
} else {
|
|
/**
|
|
* only use existing keys
|
|
*/
|
|
// console.log("only use existing keys,", pointKeyName)
|
|
if (mappingKeys.indexOf(pointKeyName) !== -1) {
|
|
let valueConvertedFromMapping = featurePoint.properties[pointKeyName]
|
|
let keyConvertedFromMapping = mappingKeys[mappingKeys.indexOf(pointKeyName)]
|
|
let mappingConfigOfTag = this.mapping_config.tags[pointKeyName]
|
|
|
|
|
|
debugLog('========== mappingConfigOfTag', mappingConfigOfTag)
|
|
debugLog('convertProperty: found element', pointKeyName, '=>', keyConvertedFromMapping, 'value : ', valueConvertedFromMapping)
|
|
let convertedValue = originalValue
|
|
|
|
let typeOfConfigForKey = typeof mappingConfigOfTag
|
|
// console.log('typeOfConfigForKey', typeOfConfigForKey)
|
|
let isStringValue = typeOfConfigForKey === 'string'
|
|
let isConfigMappingObject = typeOfConfigForKey === 'object'
|
|
|
|
debugLog('convertProperty: - typeofValue', typeOfConfigForKey)
|
|
debugLog('convertProperty: - pointKeyName', pointKeyName)
|
|
debugLog('convertProperty: - valueConvertedFromMapping', valueConvertedFromMapping)
|
|
debugLog('typeof valueConvertedFromMapping === \'string\'', typeOfConfigForKey)
|
|
|
|
|
|
debugLog('convertProperty: isStringValue?', valueConvertedFromMapping, isStringValue)
|
|
debugLog('convertProperty: isStringValue?', valueConvertedFromMapping, isStringValue)
|
|
|
|
debugLog('mappingConfigOfTag', mappingConfigOfTag)
|
|
debugLog('typeOfConfigForKey', typeOfConfigForKey)
|
|
|
|
|
|
/**
|
|
* conversion si la clé à une config d'une string, on ne change que la clé, pas la valeur
|
|
*/
|
|
if (isStringValue) {
|
|
debugLog('convertProperty: -- string value')
|
|
debugLog('convertProperty: -- string value')
|
|
debugLog('convertProperty: -- simple conversion : ', pointKeyName, '=> ', mappingConfigOfTag, '_', originalValue, '=>', valueConvertedFromMapping)
|
|
debugLog('convertProperty: -- convertedValue', convertedValue)
|
|
|
|
convertedValue = valueConvertedFromMapping
|
|
|
|
if (convertedValue) {
|
|
newProperties[mappingConfigOfTag] = convertedValue
|
|
}
|
|
} else {
|
|
debugLog('convertProperty: no string value')
|
|
}
|
|
|
|
let configObject = mappingConfigOfTag
|
|
|
|
if (isConfigMappingObject) {
|
|
|
|
|
|
// console.log('convertProperty: is config object', configObject)
|
|
let newKey: any = '' + pointKeyName
|
|
|
|
|
|
if (configObject.key_converted) {
|
|
newKey = configObject.key_converted
|
|
debugLog('key_converted newKey', newKey)
|
|
}
|
|
if (configObject.transform_function) {
|
|
convertedValue = configObject.transform_function(originalValue)
|
|
}
|
|
|
|
|
|
if (configObject.truthy_value) {
|
|
// convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value
|
|
// exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1".
|
|
// on met donc truthy_value: '1'
|
|
|
|
debugLog('truthy_value', originalValue)
|
|
if (custom_utils.truthyValues.indexOf(originalValue) !== -1) {
|
|
convertedValue = configObject.truthy_value
|
|
}
|
|
}
|
|
if (configObject.falsy_value) {
|
|
if (custom_utils.falsyValues.indexOf(originalValue) !== -1) {
|
|
convertedValue = configObject.falsy_value
|
|
}
|
|
}
|
|
|
|
/**
|
|
* conversion booléenne
|
|
*/
|
|
if (mappingValueObject.keep_only_truthy_yes_or_no_without_enum) {
|
|
if (originalValue.indexOf(";") === -1 && custom_utils.truthyValues.indexOf(originalValue) !== -1) {
|
|
convertedValue = 'yes'
|
|
} else {
|
|
|
|
remove_original_key = true
|
|
}
|
|
|
|
}
|
|
|
|
if (mappingValueObject.convert_to_boolean_value) {
|
|
|
|
// debugLog('convertProperty: is boolean_value_conversion')
|
|
|
|
convertedValue = custom_utils.convertToYesOrNo(originalValue)
|
|
|
|
// if (pointKeyName === 'prise_type_2') {
|
|
// console.log('convertProperty: is boolean_value_conversion', pointKeyName, keyConvertedFromMapping, originalValue, '=>', convertedValue)
|
|
|
|
// newProperties[keyConvertedFromMapping] = convertedValue
|
|
// }
|
|
|
|
// console.log('convertProperty: is boolean_value_conversion', pointKeyName, keyConvertedFromMapping, originalValue, '=>', convertedValue)
|
|
if (configObject.ignore_if_falsy && !custom_utils.convertToBoolean(originalValue)) {
|
|
return newProperties
|
|
}
|
|
if (configObject.ignore_if_truthy && custom_utils.convertToBoolean(originalValue)) {
|
|
return newProperties
|
|
}
|
|
|
|
} else {
|
|
debugLog('convertProperty: is NOT having boolean_value_conversion', mappingValueObject)
|
|
}
|
|
if (configObject.invert_boolean_value) {
|
|
convertedValue = !custom_utils.convertToBoolean(originalValue) ? 'yes' : 'no'
|
|
debugLog('invert boolean', convertedValue, originalValue)
|
|
}
|
|
|
|
|
|
|
|
// gestion des puissances de bornes
|
|
// avec une fonction de transformation des valeurs
|
|
// parmi le domaine du jeu de données
|
|
// nécessite une clé conditionnelle à la valeur true d'autres clés converties.
|
|
if (configObject.keep_only_max_in_enum) {
|
|
let max = custom_utils.find_max_in_string(originalValue)
|
|
if (max && max < 401) {
|
|
convertedValue = max + ' kW'
|
|
} else {
|
|
convertedValue = originalValue
|
|
}
|
|
}
|
|
if (configObject.socket_output_find_correspondances) {
|
|
|
|
convertedValue = detectSocketOutputFromFeaturePoint({
|
|
pointKeyName, mappingKeys, featurePoint, newProperties, originalValue
|
|
})
|
|
|
|
}
|
|
|
|
if (configObject.remove_stars) {
|
|
// Remplace toutes les occurrences de * de manière greedy
|
|
convertedValue = originalValue.replace('*', '')
|
|
debugLog('remove_stars', convertedValue, originalValue)
|
|
}
|
|
|
|
|
|
if (configObject.convert_to_phone) {
|
|
convertedValue = Formatters.convertToPhone(originalValue)
|
|
if (originalValue !== convertedValue) {
|
|
this.stats.phones_updated++
|
|
this.stats.phones_updated_list.push(convertedValue)
|
|
} else {
|
|
this.stats.phones_not_updated++
|
|
}
|
|
debugLog('convertedValue convert_to_phone', originalValue, '=>', convertedValue)
|
|
|
|
}
|
|
if (configObject.convert_to_name) {
|
|
convertedValue = Formatters.convertToName(originalValue)
|
|
}
|
|
if (configObject.remove_original_key) {
|
|
remove_original_key = true
|
|
}
|
|
|
|
|
|
if (configObject.ignore_if_falsy && (false === custom_utils.convertToBoolean(originalValue))) {
|
|
remove_original_key = true
|
|
}
|
|
if (configObject.ignore_if_truthy && custom_utils.convertToBoolean(originalValue)) {
|
|
remove_original_key = true
|
|
}
|
|
|
|
if (configObject.truncate_enums_to_limit) {
|
|
// console.log('configObject.truncate_enums_to_limit', configObject)
|
|
convertedValue = custom_utils.truncate_enums_to_limit(convertedValue, configObject.truncate_enums_to_limit)
|
|
debugLog('truncate_enums_to_limit => ', convertedValue)
|
|
}
|
|
|
|
let conditionalConfig: any = ''
|
|
|
|
// console.log('/////////// configObject.ignore_if_falsy', configObject.ignore_if_falsy, "converti en booléen", custom_utils.convertToBoolean(originalValue), "remove?", remove_original_key)
|
|
|
|
|
|
/**
|
|
* config pour une clé
|
|
* nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser
|
|
*/
|
|
|
|
if (!remove_original_key && configObject.conditional_values) {
|
|
|
|
// convert numbers from json to string to compare them correctly
|
|
originalValue = '' + originalValue
|
|
|
|
let keysConditionnalValues: any = Object.keys(configObject.conditional_values)
|
|
let foundValue = keysConditionnalValues.indexOf(originalValue)
|
|
conditionalConfig = configObject.conditional_values[keysConditionnalValues[foundValue]]
|
|
|
|
// par défaut on retire les valeurs qui ne sont pas dans la liste des valeurs conditionnelles
|
|
// sauf si on a activé l'option allow_unspecified_conditional_values dans la MappingConfigType
|
|
if (!this.mapping_config.allow_unspecified_conditional_values && foundValue === -1) {
|
|
|
|
// console.log('!!!!!!!!!! (x) => ignore', originalValue, ' in ', pointKeyName, 'on vire ', keyConvertedFromMapping)
|
|
remove_original_key = true
|
|
}
|
|
|
|
|
|
if (!remove_original_key) {
|
|
|
|
// console.log('-------- on garde la valeur', originalValue, ' dans ', pointKeyName)
|
|
|
|
if (foundValue !== -1) {
|
|
debugLog('found condition', foundValue)
|
|
|
|
/** ----------------------
|
|
* gestion des valeurs conditionnelles
|
|
* ---------------------- */
|
|
debugLog('conditionnalConfig', conditionalConfig)
|
|
|
|
|
|
if (conditionalConfig.ignore_this_data) {
|
|
debugLog(`on ignore cette clé car sa valeur "${originalValue}" est à exclure: `, pointKeyName, '=>', newKey)
|
|
remove_original_key = true;
|
|
}
|
|
|
|
let lowerKey = (originalValue + '').toLowerCase()
|
|
if (conditionalConfig.truthy_value) {
|
|
// convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value
|
|
// exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1".
|
|
// on met donc truthy_value: '1'
|
|
if (custom_utils.truthyValues.indexOf(lowerKey) !== -1) {
|
|
convertedValue = conditionalConfig.truthy_value
|
|
}
|
|
}
|
|
if (conditionalConfig.falsy_value) {
|
|
if (custom_utils.falsyValues.indexOf(lowerKey) !== -1) {
|
|
convertedValue = conditionalConfig.falsy_value
|
|
}
|
|
}
|
|
|
|
// use the value converted
|
|
if (conditionalConfig.value_converted) {
|
|
convertedValue = conditionalConfig.value_converted
|
|
}
|
|
|
|
|
|
|
|
}
|
|
// console.log('convertedValue =>', convertedValue)
|
|
if (conditionalConfig?.tags_to_add) {
|
|
debugLog('on ajoute des tags', conditionalConfig.tags_to_add)
|
|
// on peut définir un ensemble de tags à rajouter
|
|
let tagKeys = Object.keys(conditionalConfig.tags_to_add)
|
|
debugLog('conditionnalConfig.tags_to_add', conditionalConfig.tags_to_add)
|
|
tagKeys.forEach((index: any) => {
|
|
debugLog('key', index)
|
|
debugLog('value', conditionalConfig.tags_to_add[index])
|
|
newProperties[index] = conditionalConfig.tags_to_add[index]
|
|
})
|
|
}
|
|
}
|
|
|
|
|
|
} else {
|
|
debugLog('no conditional values', configObject)
|
|
}
|
|
|
|
debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}')
|
|
debugLog(' =============== remove_original_key', newKey, remove_original_key)
|
|
|
|
let keysOfConfigObject: string[] = [];
|
|
let hasKeyIgnoreThisData = false;
|
|
if (configObject) {
|
|
keysOfConfigObject = Object.keys(configObject)
|
|
debugLog('keysOfConfigObject', keysOfConfigObject)
|
|
hasKeyIgnoreThisData = (keysOfConfigObject.indexOf('ignore_this_data') !== -1)
|
|
// console.log('-------- hasKeyIgnoreThisData', hasKeyIgnoreThisData)
|
|
}
|
|
|
|
debugLog('remove_original_key && newKey && convertedValue && hasKeyIgnoreThisData', remove_original_key, newKey, convertedValue, hasKeyIgnoreThisData)
|
|
|
|
// console.log('newKey && convertedValue && !hasKeyIgnoreThisData', newKey && convertedValue && !hasKeyIgnoreThisData, newKey, convertedValue, !hasKeyIgnoreThisData)
|
|
|
|
// failsafe for boolean values, OSM never returns true or false
|
|
if (convertedValue == 'true') {
|
|
convertedValue = 'yes'
|
|
}
|
|
if (convertedValue == 'false') {
|
|
convertedValue = 'no'
|
|
}
|
|
|
|
if (newKey && convertedValue && !hasKeyIgnoreThisData && !remove_original_key
|
|
) {
|
|
debugLog('convertedValue', convertedValue)
|
|
|
|
debugLog('convertProperty: added', newKey, (`${convertedValue}`).trim())
|
|
newProperties[newKey] = (`${convertedValue}`).trim()
|
|
}
|
|
if (remove_original_key) {
|
|
// console.log('remove_original_key', pointKeyName, originalValue, '=>', convertedValue)
|
|
delete newProperties[pointKeyName];
|
|
}
|
|
|
|
}
|
|
else {
|
|
debugLog('!!!!!! not isConfigMappingObject: ', isConfigMappingObject)
|
|
}
|
|
} else {
|
|
debugLog('!!!!!! property not found in mappingKeys: ', pointKeyName)
|
|
}
|
|
}
|
|
|
|
return newProperties;
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|