mirror of
https://forge.chapril.org/tykayn/wololo
synced 2025-06-20 01:34:42 +02:00
add tests and utils : limit values, get max in enums
This commit is contained in:
Tykayn
tykayn
parent
aa35803a0b
commit
b4c28335b2
10 changed files with 153 additions and 43 deletions
|
|
@ -92,7 +92,7 @@ function updateMainFile(configName: string): void {
|
|||
let mainFileContent = fs.readFileSync(mainFilePath, 'utf8');
|
||||
|
||||
// Ajouter l'import
|
||||
const importStatement = `import ${configName} from './mappings/converters/config${configName.replace(/^Mapping/, '')}'\n`;
|
||||
const importStatement = `import ${configName} from './mappings/converters/config${configName.replaceAll(/^Mapping/, '')}'\n`;
|
||||
const importInsertPos = mainFileContent.indexOf('const limitWarningPercentageChangeInPoints');
|
||||
|
||||
if (importInsertPos !== -1) {
|
||||
|
|
@ -143,7 +143,7 @@ async function main() {
|
|||
}
|
||||
|
||||
// Générer le nom du fichier
|
||||
const fileName = `config${configName.replace(/^Mapping/, '')}.ts`;
|
||||
const fileName = `config${configName.replaceAll(/^Mapping/, '')}.ts`;
|
||||
const filePath = path.join(configDir, fileName);
|
||||
|
||||
// Vérifier si le fichier existe déjà
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ function csvToGeoJSON(options: Options): FeatureCollection<Point> {
|
|||
let headers: string[] = [];
|
||||
let headersFound = false;
|
||||
|
||||
let limitOffset = 30000000;
|
||||
let limitOffset = 100000000;
|
||||
|
||||
fs.createReadStream(filePath)
|
||||
.pipe(csvParser({ headers: hasHeaders }))
|
||||
|
|
|
|||
4
makefile
4
makefile
|
|
@ -33,4 +33,6 @@ osmose_irve:
|
|||
panneaux:
|
||||
npx ts-node unzip_csv.ts -u https://www.data.gouv.fr/fr/datasets/r/16a20f4e-0c06-40ff-adda-54f214099e5f -o panneaux_limite_de_vitesse_fr_panoramax_detections.csv -z
|
||||
npx ts-node csv_to_geojson.ts -d etalab_data/panneaux -f panneaux_limite_de_vitesse_fr_panoramax_detections.csv --latColumn 'GPSLatitude' --lonColumn 'GPSLongitude' -h
|
||||
ts-node convert_to_osm_tags.ts --source etalab_data/panneaux/panneaux_limite_de_vitesse_fr_panoramax_detections.csv.geojson --output-file=panneaux_maxspeed_panoramax.json --engine-config=MappingPanneauxMaxSpeed
|
||||
ts-node convert_to_osm_tags.ts --source etalab_data/panneaux/panneaux_limite_de_vitesse_fr_panoramax_detections.csv.geojson --output-file=panneaux_maxspeed_panoramax.json --engine-config=MappingPanneauxMaxSpeed
|
||||
tests:
|
||||
npm test
|
||||
|
|
@ -19,6 +19,7 @@ const MappingIRVE: MappingConfigType = {
|
|||
* select only certain points from the source
|
||||
*/
|
||||
filters: {
|
||||
// offset: 1, // limiter à une feature pour faire des tests
|
||||
enable_coordinates_filter: false,
|
||||
enable_properties_filter: true,
|
||||
// filter_points_older_than_year: 2024,
|
||||
|
|
@ -60,11 +61,12 @@ const MappingIRVE: MappingConfigType = {
|
|||
key_converted: 'operator:phone',
|
||||
convert_to_phone: true, // conversion en format international si possible
|
||||
},
|
||||
contact_operateur: 'operator:email', // ici, on souhaite convertir la clé contact_operateur=bidule en email=bidule
|
||||
contact_operateur: 'operator:email',
|
||||
|
||||
id_station_itinerance: {
|
||||
key_converted: 'ref:EU:EVSE',
|
||||
remove_stars: true,
|
||||
truncate_enums_to_limit: 255,
|
||||
},
|
||||
|
||||
id_station_local: 'ref',
|
||||
|
|
@ -84,6 +86,7 @@ const MappingIRVE: MappingConfigType = {
|
|||
|
||||
},
|
||||
reservation: {
|
||||
key_converted: 'reservation',
|
||||
convert_to_boolean_value: true, // convertit en yes ou no
|
||||
},
|
||||
// observations: 'note',
|
||||
|
|
@ -125,6 +128,7 @@ const MappingIRVE: MappingConfigType = {
|
|||
},
|
||||
// ******** champs plus complexes
|
||||
horaires: 'opening_hours', // déjà au bon format, enfin, en général. vérifier avec le validateur josm.
|
||||
tarification: 'charge',
|
||||
paiement_cb: {
|
||||
key_converted: 'payment:credit_cards',
|
||||
// ignore_if_falsy: true,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import custom_utils from './utils'
|
|||
import MappingConfigType from "./mapping-config.type";
|
||||
import Formatters from "./formatters";
|
||||
|
||||
const {debugLog} = custom_utils
|
||||
const { debugLog, find_max_in_string, truncate_enums_to_limit } = custom_utils
|
||||
|
||||
let listOfBooleanKeys = [
|
||||
"prise_type_ef",
|
||||
|
|
@ -57,7 +57,7 @@ export default class {
|
|||
let geoJSONConvertedPoint: any = {}
|
||||
|
||||
|
||||
geoJSONConvertedPoint.properties = {...this.mapping_config.default_properties_of_point}
|
||||
geoJSONConvertedPoint.properties = { ...this.mapping_config.default_properties_of_point }
|
||||
geoJSONConvertedPoint.type = featurePointGeoJson.type
|
||||
geoJSONConvertedPoint.geometry = featurePointGeoJson.geometry
|
||||
|
||||
|
|
@ -130,7 +130,7 @@ export default class {
|
|||
return newList;
|
||||
}
|
||||
|
||||
filterListOfPointsByExcludingIfColumnBeforeSomeYear(year: number, column:string, list_of_points: any[]): any[] {
|
||||
filterListOfPointsByExcludingIfColumnBeforeSomeYear(year: number, column: string, list_of_points: any[]): any[] {
|
||||
let newList: any[] = []
|
||||
list_of_points.forEach((geojsonPoint: any) => {
|
||||
let pointProperties = Object.keys(geojsonPoint.properties)
|
||||
|
|
@ -138,7 +138,7 @@ export default class {
|
|||
|
||||
// on inclut les points dont la date de création est de l'année demandée ou supérieure
|
||||
if (pointProperties.includes(column) &&
|
||||
(geojsonPoint.properties[column].substring(0,4) * 1) >= year
|
||||
(geojsonPoint.properties[column].substring(0, 4) * 1) >= year
|
||||
) {
|
||||
newList.push(geojsonPoint)
|
||||
}
|
||||
|
|
@ -155,9 +155,12 @@ export default class {
|
|||
// trouver la valeur
|
||||
// socket_output_find_correspondances
|
||||
if (pointProperties.includes('puissance_nominale') &&
|
||||
1 * (geojsonPoint.properties['puissance_nominale'].replace(' kW', '')) > minValue
|
||||
1 * (geojsonPoint.properties['puissance_nominale'].replaceAll(' kW', '')) > minValue
|
||||
) {
|
||||
newList.push(geojsonPoint)
|
||||
let max_power = find_max_in_string(geojsonPoint.properties['puissance_nominale'])
|
||||
if (max_power >= minValue) {
|
||||
newList.push(geojsonPoint)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
|
|
@ -176,28 +179,25 @@ export default class {
|
|||
|
||||
debugLog('mapElementFromConf: config_name', this.mapping_config.config_name)
|
||||
let mappingKeys = Object.keys(this.mapping_config.tags)
|
||||
let featurePointPropertiesKeys = []
|
||||
let featurePointPropertiesKeys: string[] = [];
|
||||
if (this.mapping_config.osmose) {
|
||||
// only creation of new points are handled by now [2023-10-07]
|
||||
featurePointPropertiesKeys = Object.keys(featurePoint.properties.fixes[0][0].create)
|
||||
// debugLog('featurePointPropertiesKeys', featurePointPropertiesKeys)
|
||||
|
||||
featurePointPropertiesKeys = Object.keys(featurePoint.properties.fixes[0][0].create);
|
||||
} else {
|
||||
featurePointPropertiesKeys = Object.keys(featurePoint.properties)
|
||||
featurePointPropertiesKeys = Object.keys(featurePoint.properties);
|
||||
}
|
||||
|
||||
|
||||
debugLog('mapElementFromConf: ============= keys mappingKeys:', this.mapping_config.tags.length, mappingKeys.length)
|
||||
debugLog('mapElementFromConf: ============= keys featurePointPropertiesKeys :', featurePoint.properties.length, featurePointPropertiesKeys.length)
|
||||
|
||||
let newProperties = {...this.mapping_config.default_properties_of_point}
|
||||
let newProperties = { ...this.mapping_config.default_properties_of_point }
|
||||
|
||||
|
||||
// reinit properties of current point
|
||||
let basePoint = Object.create(featurePoint)
|
||||
basePoint.type = featurePoint.type
|
||||
basePoint.geometry = featurePoint.geometry
|
||||
basePoint.properties = {...this.mapping_config.default_properties_of_point}
|
||||
basePoint.properties = { ...this.mapping_config.default_properties_of_point }
|
||||
|
||||
// apply new properties if found in mapping config
|
||||
featurePointPropertiesKeys.forEach(pointKeyName => {
|
||||
|
|
@ -279,6 +279,7 @@ export default class {
|
|||
let keyConvertedFromMapping = mappingKeys[mappingKeys.indexOf(pointKeyName)]
|
||||
let mappingConfigOfTag = this.mapping_config.tags[pointKeyName]
|
||||
|
||||
|
||||
debugLog('========== mappingConfigOfTag', mappingConfigOfTag)
|
||||
debugLog('convertProperty: found element', pointKeyName, '=>', keyConvertedFromMapping, 'value : ', valueConvertedFromMapping)
|
||||
let convertedValue = originalValue
|
||||
|
|
@ -387,13 +388,12 @@ export default class {
|
|||
}
|
||||
// ajouter les tags de socket newProperties
|
||||
|
||||
let converted_value = originalValue.replace(/[^\d\.\,]/g, '').replace(',', '.')
|
||||
let converted_value = find_max_in_string(originalValue.replaceAll('.00', '').replaceAll(',', '.'))
|
||||
let max_output = 401
|
||||
// do not limit accepted values
|
||||
let out = ''
|
||||
|
||||
if (intOriginalValue < max_output) {
|
||||
|
||||
// rajouter l'unité de puissance kW dans la valeur
|
||||
out = converted_value + ' kW'
|
||||
|
||||
|
|
@ -402,13 +402,13 @@ export default class {
|
|||
debugLog('too high kW value detected', originalValue)
|
||||
if (intOriginalValue > 1000 && intOriginalValue < 401000) {
|
||||
|
||||
let kilowatts = (parseFloat(converted_value) / 1000).toFixed(2).replace('.00', '');
|
||||
out = ('' + kilowatts + ' kW').replace('.00', '')
|
||||
let kilowatts = (converted_value / 1000).toFixed(2);
|
||||
out = ('' + kilowatts + ' kW').replaceAll('.00', '')
|
||||
debugLog('valeurs en Watts out', out, 'original:', originalValue)
|
||||
this.stats.power_output++
|
||||
}
|
||||
}
|
||||
out = (out).replace('.00', '')
|
||||
out = (out).replaceAll('.00', '')
|
||||
|
||||
// debug land
|
||||
if (has_prise_type_combo_ccs) {
|
||||
|
|
@ -453,6 +453,7 @@ export default class {
|
|||
debugLog('no sockets', this.current_geojson_point.properties.ref)
|
||||
}
|
||||
}
|
||||
convertedValue = out;
|
||||
return out
|
||||
|
||||
|
||||
|
|
@ -462,7 +463,8 @@ export default class {
|
|||
debugLog('invert boolean', convertedValue, originalValue)
|
||||
}
|
||||
if (configObject.remove_stars) {
|
||||
convertedValue = originalValue.replace('*', '')
|
||||
// Remplace toutes les occurrences de * de manière greedy
|
||||
convertedValue = originalValue.replaceAll('*', '')
|
||||
debugLog('remove_stars', convertedValue, originalValue)
|
||||
}
|
||||
|
||||
|
|
@ -490,6 +492,13 @@ export default class {
|
|||
if (configObject.ignore_if_truthy && this.truthyValues.indexOf(originalValue) !== -1) {
|
||||
remove_original_key = true
|
||||
}
|
||||
|
||||
if (configObject.truncate_enums_to_limit) {
|
||||
// console.log('configObject.truncate_enums_to_limit', configObject)
|
||||
convertedValue = custom_utils.truncate_enums_to_limit(convertedValue, configObject.truncate_enums_to_limit)
|
||||
debugLog('truncate_enums_to_limit => ', convertedValue)
|
||||
}
|
||||
|
||||
/**
|
||||
* config pour une clé
|
||||
* nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser
|
||||
|
|
@ -570,7 +579,7 @@ export default class {
|
|||
debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}')
|
||||
debugLog(' =============== remove_original_key', newKey, remove_original_key)
|
||||
|
||||
let keysOfConfigObject = [];
|
||||
let keysOfConfigObject: string[] = [];
|
||||
let hasKeyIgnoreThisData = false;
|
||||
if (configObject) {
|
||||
keysOfConfigObject = Object.keys(configObject)
|
||||
|
|
|
|||
|
|
@ -1,26 +1,26 @@
|
|||
import custom_utils from "./utils";
|
||||
|
||||
const {debugLog,prefix_phone_fr_only} = custom_utils
|
||||
const { debugLog, prefix_phone_fr_only } = custom_utils
|
||||
|
||||
/**
|
||||
* Class that helps to convert values into predefined formats
|
||||
*/
|
||||
export default class Formatters {
|
||||
|
||||
static convertToPhone(originalValue: string) :string{
|
||||
static convertToPhone(originalValue: string): string {
|
||||
/**
|
||||
* nettoyer les numéros de téléphone en ne gardant que les nombres et le préfixe de pays
|
||||
*/
|
||||
debugLog("convertToPhone:" , originalValue);
|
||||
debugLog("convertToPhone:", originalValue);
|
||||
// debugLog('originalValue', originalValue.substring(1))
|
||||
if (!originalValue) {
|
||||
originalValue = ''
|
||||
}
|
||||
let original_without_spaces = originalValue.replace(' ', '')
|
||||
let original_without_spaces = originalValue.replaceAll(' ', '')
|
||||
let cleaned_value = `${original_without_spaces}`
|
||||
cleaned_value = cleaned_value
|
||||
.trim()
|
||||
.replace('Stations-e', '')
|
||||
.replaceAll('Stations-e', '')
|
||||
.replace(/[a-zA-Zéèà]/ig, '')
|
||||
.replace(/[\(\)\.\- ]/g, '')
|
||||
let add_prefix = false;
|
||||
|
|
@ -30,9 +30,9 @@ export default class Formatters {
|
|||
) {
|
||||
add_prefix = true
|
||||
}
|
||||
cleaned_value = cleaned_value.replace('+33', '')
|
||||
cleaned_value = cleaned_value.replaceAll('+33', '')
|
||||
|
||||
debugLog("convertToPhone: cleaned_value" , cleaned_value);
|
||||
debugLog("convertToPhone: cleaned_value", cleaned_value);
|
||||
|
||||
if (/^0/.test(cleaned_value)) {
|
||||
cleaned_value = cleaned_value.substring(1)
|
||||
|
|
@ -55,7 +55,7 @@ export default class Formatters {
|
|||
})
|
||||
|
||||
|
||||
convertedValue = convertedValue.replace(' ', ' ').trim();
|
||||
convertedValue = convertedValue.replaceAll(' ', ' ').trim();
|
||||
debugLog('convertedValue', convertedValue)
|
||||
if (
|
||||
/^\d/.test(convertedValue) &&
|
||||
|
|
@ -70,7 +70,7 @@ export default class Formatters {
|
|||
debugLog('phone: ', originalValue, '=>', convertedValue)
|
||||
|
||||
|
||||
return ""+convertedValue;
|
||||
return "" + convertedValue;
|
||||
}
|
||||
|
||||
static convertToName(originalValue: string) {
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@ export interface FeaturePropertyMappingConfigType {
|
|||
ignore_if_falsy?: boolean,
|
||||
ignore_if_truthy?: boolean,
|
||||
remove_stars?: boolean,
|
||||
truncate_enums_to_limit?: number,
|
||||
conditional_values?: ConditionnalValuesConfigType,
|
||||
transform_function?: Function,
|
||||
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ function writeFile(fileName: string, fileContent: any, outputPathOverride: strin
|
|||
console.log('pas de output', outputPathOverride
|
||||
)
|
||||
}
|
||||
let destination = `./${output_folder}/${fileName}`.replace('//', '/');
|
||||
let destination = `./${output_folder}/${fileName}`.replaceAll('//', '/');
|
||||
console.log('write file ', destination)
|
||||
return fs.writeFile(
|
||||
destination,
|
||||
|
|
@ -70,9 +70,61 @@ function writeFile(fileName: string, fileContent: any, outputPathOverride: strin
|
|||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* trouve le max dans une string séparée par des points-virgules
|
||||
* @param str
|
||||
* @returns
|
||||
*/
|
||||
function find_max_in_string(str: string, separator: string = ';') {
|
||||
let max = 0;
|
||||
if (str.indexOf(separator) !== -1) {
|
||||
let explode = str.split(separator);
|
||||
explode.forEach((item: string) => {
|
||||
// keep only the number
|
||||
let value = parseInt(item.replaceAll(/[^0-9]/g, ''));
|
||||
if (value && value > max) {
|
||||
max = value;
|
||||
}
|
||||
});
|
||||
return max;
|
||||
}
|
||||
return parseInt(str.replaceAll(/[^0-9]/g, ''));
|
||||
}
|
||||
|
||||
/**
|
||||
* tronque une string à une longueur donnée
|
||||
* @param str
|
||||
* @param limit
|
||||
* @returns
|
||||
*/
|
||||
function truncate_enums_to_limit(str: string, limit: number = 255) {
|
||||
if (str.indexOf(';') !== -1) {
|
||||
let elements = str.split(';');
|
||||
let result: string[] = [];
|
||||
let currentLength = 0;
|
||||
|
||||
for (let element of elements) {
|
||||
// +1 pour le point-virgule qui sera ajouté
|
||||
if (currentLength + element.length + 1 <= limit) {
|
||||
result.push(element);
|
||||
currentLength += element.length + 1;
|
||||
}
|
||||
|
||||
}
|
||||
let joined = result.join(';');
|
||||
return joined;
|
||||
}
|
||||
if (str.length > limit) {
|
||||
return str.substring(0, limit)
|
||||
}
|
||||
return str
|
||||
}
|
||||
|
||||
export default {
|
||||
debugLog,
|
||||
isBooleanKey,
|
||||
writeFile,
|
||||
prefix_phone_fr_only
|
||||
find_max_in_string,
|
||||
truncate_enums_to_limit,
|
||||
prefix_phone_fr_only,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import {
|
|||
mappingName,
|
||||
mappingSame, mappingTruthy, mappingFalsy, mappingIgnoreFalsy, mappingIgnoreTruthy
|
||||
} from './data/mappings_to_test'
|
||||
import utils from '../mappings/utils'
|
||||
import Formatters from '../mappings/formatters'
|
||||
|
||||
const testingGeoJson = require('./data/testing.json')
|
||||
|
||||
|
|
@ -29,10 +31,10 @@ describe('mapping properties with rich mapping engine', () => {
|
|||
let newProperties = Mapping_engine.convertProperty('equal',
|
||||
Object.keys(mappingSame.tags),
|
||||
feature_to_test,
|
||||
mappingSame.default_properties_of_point )
|
||||
mappingSame.default_properties_of_point)
|
||||
|
||||
expect(newProperties).toStrictEqual({
|
||||
equal : "same value"
|
||||
equal: "same value"
|
||||
})
|
||||
})
|
||||
test('retrieve config name in mapping engine', () => {
|
||||
|
|
@ -41,11 +43,11 @@ describe('mapping properties with rich mapping engine', () => {
|
|||
})
|
||||
test('maps nom_amenageur to name, and keep the same value', () => {
|
||||
let Mapping_engine = new mapping_engine(mappingName)
|
||||
let newProperties = Mapping_engine.convertProperty('nom_amenageur',Object.keys(mappingName.tags),feature_to_test,mappingName.default_properties_of_point )
|
||||
let newProperties = Mapping_engine.convertProperty('nom_amenageur', Object.keys(mappingName.tags), feature_to_test, mappingName.default_properties_of_point)
|
||||
expect(Mapping_engine.getConfig().config_name).toBe('testing config mappingName')
|
||||
|
||||
expect(newProperties).toStrictEqual({
|
||||
name : "Bob"
|
||||
name: "Bob"
|
||||
})
|
||||
})
|
||||
test('ignore one value if it is truthy', () => {
|
||||
|
|
@ -123,7 +125,7 @@ describe('mapping properties with rich mapping engine', () => {
|
|||
|
||||
feature_to_test.properties.telephone_operateur = 'Stations-e'
|
||||
mapped_point = Mapping_engine.mapElementFromConf(feature_to_test)
|
||||
expect(mapped_point.properties).toStrictEqual({ })
|
||||
expect(mapped_point.properties).toStrictEqual({})
|
||||
|
||||
feature_to_test.properties.telephone_operateur = '+33 0 7 66 38 74 96'
|
||||
expected_converted_phone = '+33 7 66 38 74 96'
|
||||
|
|
@ -162,3 +164,43 @@ xdescribe('infer domain of values from geojson file', () => {
|
|||
xdescribe('build mapping engine config from unique values', () => {
|
||||
test('builds a valid mapping config', () => { })
|
||||
})
|
||||
|
||||
describe('find max in enum', () => {
|
||||
test('value has enums', () => {
|
||||
let value = '1;2;3;10;4;5;6;7;8;9;'
|
||||
let max = utils.find_max_in_string(value)
|
||||
expect(max).toBe(10)
|
||||
})
|
||||
test('value has enums with maybe units', () => {
|
||||
let value = '1;2;3;10 km;4;5 kw;6;7;8;9;'
|
||||
let max = utils.find_max_in_string(value)
|
||||
expect(max).toBe(10)
|
||||
})
|
||||
|
||||
test('value has no enums', () => {
|
||||
let max = utils.find_max_in_string('10')
|
||||
console.log('max', max)
|
||||
expect(max).toBe(10)
|
||||
})
|
||||
})
|
||||
|
||||
describe('truncate enums to limit', () => {
|
||||
test('truncate enums to limit', () => {
|
||||
let value = '1;2;3;10 km;4;5 kw;6;7;8;9;'
|
||||
let truncated = utils.truncate_enums_to_limit(value, 5)
|
||||
expect(truncated).toBe('1;2;')
|
||||
})
|
||||
})
|
||||
|
||||
describe('formatters tests', () => {
|
||||
test('format phone', () => {
|
||||
let phone = '+331 2345 6789'
|
||||
let formatted = Formatters.convertToPhone(phone)
|
||||
expect(formatted).toBe('+33 1 23 45 67 89')
|
||||
})
|
||||
test('format name', () => {
|
||||
let name = 'bob lénon'
|
||||
let formatted = Formatters.convertToName(name)
|
||||
expect(formatted).toBe('Bob lénon')
|
||||
})
|
||||
})
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# chemin du dossier à parcourir
|
||||
source functions.sh
|
||||
dir_to_search="../mappings/extractors"
|
||||
dir_to_search="mappings/extractors"
|
||||
|
||||
# recherche tous les fichiers .sh dans le dossier et ses sous-dossiers
|
||||
find "$dir_to_search" -type f -name "*.sh" -print0 | while IFS= read -r -d '' file; do
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue