mirror of
https://forge.chapril.org/tykayn/wololo
synced 2025-06-20 01:34:42 +02:00
add config plaques commémoratives
This commit is contained in:
parent
2956a3ffee
commit
c9ecd31302
14 changed files with 536 additions and 166 deletions
17
README.md
17
README.md
|
@ -102,12 +102,13 @@ Données possible d'intégrer en plus avec le même jeu de données:
|
|||
* ne pas mettre ref:EU:EVSE si la valeur vaut "non concerné"
|
||||
* ajouter les références européennes ref:EU:EVSE.
|
||||
* l'accessibilité => wheelchair pour certaines valeurs limitées.
|
||||
### Points d'apport volontaire
|
||||
### Toilettes publiques
|
||||
### Défibrillateurs - DAE
|
||||
### Centres de santé
|
||||
### Bus
|
||||
|
||||
## projets à venir
|
||||
### Référentiel national des bâtiments RNB
|
||||
# Thématiques
|
||||
- Points d'apport volontaire
|
||||
- Toilettes publiques
|
||||
- Défibrillateurs - DAE
|
||||
- Centres de santé
|
||||
- Bus
|
||||
- Référentiel national des bâtiments RNB
|
||||
https://rnb.beta.gouv.fr
|
||||
|
||||
- plaques commémoratives
|
36
config.ts
36
config.ts
|
@ -25,8 +25,28 @@ import mappingIssy2Roues from './mappings/converters/configIssy_OpenData_2roues'
|
|||
import mappingConfigIRVEFromOsmose from './mappings/converters/configIRVE_osmose'
|
||||
import mappingConfigIRVE_simple from './mappings/converters/mappingConfigIRVE_simple'
|
||||
import mappingTest from './mappings/converters/configTest'
|
||||
import MappingPlaquesCommémorativesParis from './mappings/converters/configPlaquesCommémorativesParis'
|
||||
|
||||
/**
|
||||
* clés booléennes
|
||||
*/
|
||||
let listOfBooleanKeys = [
|
||||
"prise_type_ef",
|
||||
"prise_type_2",
|
||||
"prise_type_combo_ccs",
|
||||
"prise_type_chademo",
|
||||
"gratuit",
|
||||
"paiement_acte",
|
||||
"paiement_cb",
|
||||
"cable_t2_attache"
|
||||
]
|
||||
let irve_max_output = 401 // limite de kW de puissance pour une borne de recharge publique
|
||||
|
||||
const limitWarningPercentageChangeInPoints = 5; // show a warning when more than N percent of the number of points changed
|
||||
|
||||
|
||||
const allowed_configs: any = {
|
||||
MappingPlaquesCommémorativesParis,
|
||||
MappingPanneauxMaxSpeed,
|
||||
MappingArbresRemarquablesRouen,
|
||||
MappingRnb,
|
||||
|
@ -48,22 +68,6 @@ const allowed_configs: any = {
|
|||
MappingSurveillanceRouen
|
||||
};
|
||||
|
||||
/**
|
||||
* clés booléennes
|
||||
*/
|
||||
let listOfBooleanKeys = [
|
||||
"prise_type_ef",
|
||||
"prise_type_2",
|
||||
"prise_type_combo_ccs",
|
||||
"prise_type_chademo",
|
||||
"gratuit",
|
||||
"paiement_acte",
|
||||
"paiement_cb",
|
||||
"cable_t2_attache"
|
||||
]
|
||||
let irve_max_output = 401 // limite de kW de puissance pour une borne de recharge publique
|
||||
|
||||
const limitWarningPercentageChangeInPoints = 5; // show a warning when more than N percent of the number of points changed
|
||||
|
||||
export default {
|
||||
listOfBooleanKeys,
|
||||
|
|
|
@ -13,6 +13,33 @@
|
|||
* --testingConfig : Active le mode test avec la configuration mappingTest
|
||||
*/
|
||||
|
||||
/**
|
||||
* Mesure de l'utilisation de la RAM
|
||||
*/
|
||||
const formatBytes = (bytes: number): string => {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
const k = 1024;
|
||||
const sizes = ['Bytes', 'KB', 'MB', 'GB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
||||
}
|
||||
|
||||
/**
|
||||
* Mesure du temps d'exécution
|
||||
*/
|
||||
const tempsDebut = process.hrtime();
|
||||
|
||||
process.on('exit', () => {
|
||||
const tempsFin = process.hrtime(tempsDebut);
|
||||
const tempsExecutionMs = (tempsFin[0] * 1000 + tempsFin[1] / 1000000).toFixed(2);
|
||||
const usedMemory = process.memoryUsage();
|
||||
console.log('\nUtilisation mémoire:');
|
||||
console.log('- Heap utilisé:', formatBytes(usedMemory.heapUsed)); // Tas mémoire utilisé
|
||||
console.log('- Heap total:', formatBytes(usedMemory.heapTotal)); // Tas mémoire total
|
||||
console.log('- Mémoire totale allouée:', formatBytes(usedMemory.rss)); // Resident Set Size
|
||||
console.log(`\nTemps d'exécution: ${tempsExecutionMs} ms`);
|
||||
});
|
||||
|
||||
|
||||
import * as fs from 'fs'
|
||||
|
||||
|
@ -129,7 +156,6 @@ function writeFile(fileName: string, fileContent: any) {
|
|||
fileName = '' + outname;
|
||||
}
|
||||
let write_path = `./${output_folder}/${fileName}`
|
||||
debugLog("write file \n", fileName, write_path)
|
||||
|
||||
console.log("-------- write file \n", fileName, "\n", write_path)
|
||||
|
||||
|
@ -139,7 +165,7 @@ function writeFile(fileName: string, fileContent: any) {
|
|||
'utf8',
|
||||
(err) => {
|
||||
if (err) {
|
||||
debugLog(`Error writing file: ${err}`)
|
||||
console.error(`Error writing file: ${err}`)
|
||||
} else {
|
||||
debugLog(`File ${fileName} is written successfully!`)
|
||||
}
|
||||
|
@ -189,11 +215,15 @@ function convertDataFromSource(sourceFilePath: string, mapping: MappingConfigTyp
|
|||
console.log('features count before:', list_of_points.length)
|
||||
|
||||
|
||||
let hasFilterOffset = false;
|
||||
/**
|
||||
* filtering
|
||||
* filtering ---------------------------------------------------
|
||||
* run filters before mapping OSM tags
|
||||
*/
|
||||
if (mapping.filters) {
|
||||
if (mapping.filters.offset) {
|
||||
hasFilterOffset = true;
|
||||
}
|
||||
if (mapping.filters.exclude_point_if_tag_not_empty) {
|
||||
console.log('------ filter: exclude_point_if_tag_not_empty', mapping.filters.exclude_point_if_tag_not_empty)
|
||||
list_of_points = Mapping_engine.filterListOfPointsByExcludingIfKeyFilled(list_of_points, mapping.filters.exclude_point_if_tag_not_empty)
|
||||
|
@ -208,6 +238,9 @@ function convertDataFromSource(sourceFilePath: string, mapping: MappingConfigTyp
|
|||
}
|
||||
|
||||
}
|
||||
if (hasFilterOffset) {
|
||||
list_of_points = list_of_points.splice(0, mapping?.filters?.offset)
|
||||
}
|
||||
|
||||
// for each point from the data source, filter if we take it or not
|
||||
list_of_points.forEach((feature_point: any) => {
|
||||
|
@ -244,11 +277,6 @@ function convertDataFromSource(sourceFilePath: string, mapping: MappingConfigTyp
|
|||
})
|
||||
}
|
||||
|
||||
|
||||
// debug sur un offset de 1 seul point
|
||||
// console.log('properties_list', properties_list)
|
||||
|
||||
|
||||
/**
|
||||
* exclusion des points sur un filtrage exclude_point_if_tag_not_empty
|
||||
* utile pour les analyses comportant des points déjà présents dans osm qui ont un id_osm par exemple.
|
||||
|
@ -257,9 +285,6 @@ function convertDataFromSource(sourceFilePath: string, mapping: MappingConfigTyp
|
|||
properties_keys.forEach((tagKey: string) => {
|
||||
let tagValue = properties_list[tagKey]
|
||||
if (mapping.filters?.exclude_point_if_tag_not_empty?.indexOf(tagKey) !== undefined && tagValue && tagValue !== null) {
|
||||
// console.log('la clé est elle présente dans les filtres d exclusion?', tagKey, mapping.filters?.exclude_point_if_tag_not_empty?.indexOf(tagKey))
|
||||
// console.log('tagValue', tagValue)
|
||||
// console.log('remove original' , tagKey )
|
||||
remove_original_key = true;
|
||||
}
|
||||
})
|
||||
|
@ -317,8 +342,11 @@ function convertDataFromSource(sourceFilePath: string, mapping: MappingConfigTyp
|
|||
|
||||
})
|
||||
|
||||
// fin des filtres
|
||||
|
||||
/**
|
||||
* conversion
|
||||
* conversion ---------------------------------------------------
|
||||
*
|
||||
*/
|
||||
debugLog(' after filtering, feature_points_after_filter number of points: ', feature_points_after_filter.length)
|
||||
feature_points_after_filter.forEach((feature_point: any) => {
|
||||
|
@ -343,7 +371,6 @@ function convertDataFromSource(sourceFilePath: string, mapping: MappingConfigTyp
|
|||
|
||||
debugLog('convert : convertedGeoJson.features.length', converted_geo_json.features.length)
|
||||
|
||||
console.log('converted_geo_json.features.length', converted_geo_json.features.length)
|
||||
|
||||
// write file on disk
|
||||
if (converted_geo_json.features.length) {
|
||||
|
@ -353,7 +380,7 @@ function convertDataFromSource(sourceFilePath: string, mapping: MappingConfigTyp
|
|||
output_supplement = '_' + output_supplement;
|
||||
}
|
||||
let fileNameToWrite = '_' + filteredName + output_supplement + '.geojson'
|
||||
console.log('converted features:', converted_geo_json.features.length)
|
||||
console.log('features converties:', converted_geo_json.features.length)
|
||||
console.log('différences nombre de features: ', data_transformed.features.length - converted_geo_json.features.length)
|
||||
|
||||
const percentChange = 100 - ((converted_geo_json.features.length / data_transformed.features.length) * 100);
|
||||
|
@ -365,8 +392,6 @@ function convertDataFromSource(sourceFilePath: string, mapping: MappingConfigTyp
|
|||
}
|
||||
debugLog('convert : write file ', fileNameToWrite)
|
||||
|
||||
// console.log('mapping_engine.stats', Mapping_engine.stats)
|
||||
console.log('mapping config name:', Mapping_engine.getConfig().config_name)
|
||||
writeFile(fileNameToWrite, JSON.stringify(converted_geo_json, null, 2))
|
||||
|
||||
} else {
|
||||
|
|
|
@ -92,7 +92,7 @@ function updateMainFile(configName: string): void {
|
|||
let mainFileContent = fs.readFileSync(mainFilePath, 'utf8');
|
||||
|
||||
// Ajouter l'import
|
||||
const importStatement = `import ${configName} from './mappings/converters/config${configName.replaceAll(/^Mapping/, '')}'\n`;
|
||||
const importStatement = `import ${configName} from './mappings/converters/config${configName.replace(new RegExp(/^Mapping/, 'g'), '')}'\n`;
|
||||
const importInsertPos = mainFileContent.indexOf('const limitWarningPercentageChangeInPoints');
|
||||
|
||||
if (importInsertPos !== -1) {
|
||||
|
@ -143,7 +143,7 @@ async function main() {
|
|||
}
|
||||
|
||||
// Générer le nom du fichier
|
||||
const fileName = `config${configName.replaceAll(/^Mapping/, '')}.ts`;
|
||||
const fileName = `config${configName.replace(new RegExp(/^Mapping/, 'g'), '')}.ts`;
|
||||
const filePath = path.join(configDir, fileName);
|
||||
|
||||
// Vérifier si le fichier existe déjà
|
||||
|
|
|
@ -13,13 +13,7 @@ import path from 'path';
|
|||
import minimist from 'minimist';
|
||||
import { csvToGeoJSON, checkFile, countGeoJSONFeatures } from './csv_to_geojson.utils';
|
||||
|
||||
interface CSVConversionOptions {
|
||||
dir: string;
|
||||
file: string;
|
||||
latColumn: string;
|
||||
lonColumn: string;
|
||||
hasHeaders: boolean;
|
||||
}
|
||||
import { CSVConversionOptions } from './csv_to_geojson.utils';
|
||||
|
||||
// config des arguments de conversion
|
||||
const args = minimist<CSVConversionOptions>(process.argv.slice(2), {
|
||||
|
|
|
@ -1,4 +1,3 @@
|
|||
|
||||
import { FeatureCollection, Feature, Point } from "geojson";
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
@ -10,103 +9,91 @@ let counter_features = 0;
|
|||
let counter_missing_lat = 0;
|
||||
let counter_missing_lon = 0;
|
||||
|
||||
|
||||
function csvToGeoJSON(options: Options): FeatureCollection<Point> {
|
||||
const { dir, file, latColumn, lonColumn, hasHeaders } = options;
|
||||
|
||||
console.log(`options latColumn: ${latColumn}`);
|
||||
console.log(`options lonColumn: ${lonColumn}`);
|
||||
|
||||
const filePath = path.join(dir, file);
|
||||
const features: Feature<Point>[] = [];
|
||||
let geoJSON: FeatureCollection<Point> = {
|
||||
type: 'FeatureCollection',
|
||||
features,
|
||||
};
|
||||
|
||||
|
||||
let headers: string[] = [];
|
||||
let headersFound = false;
|
||||
let limitOffset = 100000000;
|
||||
|
||||
fs.createReadStream(filePath)
|
||||
.pipe(csvParser({ headers: hasHeaders }))
|
||||
.on('data', (row) => {
|
||||
counter += 1;
|
||||
if (!headersFound && hasHeaders) {
|
||||
let keys = Object.keys(row);
|
||||
keys.forEach((key) => {
|
||||
headers.push(row[key]);
|
||||
});
|
||||
headersFound = true;
|
||||
console.log(`headers: ${headers}`);
|
||||
}
|
||||
if (counter > limitOffset) {
|
||||
return;
|
||||
}
|
||||
if (headers.indexOf(latColumn) && headers.indexOf(lonColumn)) {
|
||||
const lat = parseFloat(row['_' + headers.indexOf(latColumn)]);
|
||||
const lon = parseFloat(row['_' + headers.indexOf(lonColumn)]);
|
||||
|
||||
// Si pas d'entêtes, utiliser des noms de colonnes génériques
|
||||
if (!hasHeaders) {
|
||||
const properties: { [key: string]: any } = {};
|
||||
Object.keys(row).forEach((key, index) => {
|
||||
properties[headers[index]] = row[key];
|
||||
});
|
||||
row = properties;
|
||||
} else {
|
||||
let keys = Object.keys(row);
|
||||
// Utiliser les entêtes comme noms de propriétés
|
||||
const properties: { [key: string]: any } = {};
|
||||
headers.forEach((header, index) => {
|
||||
|
||||
properties[header] = row['_' + index];
|
||||
});
|
||||
row = properties;
|
||||
}
|
||||
|
||||
// filtrer la ligne du header si présente
|
||||
if (lon && lat) {
|
||||
if (hasHeaders && counter > 1 || !hasHeaders || counter > limitOffset) {
|
||||
features.push({
|
||||
type: 'Feature',
|
||||
geometry: {
|
||||
type: 'Point',
|
||||
coordinates: [lon, lat],
|
||||
},
|
||||
properties: row,
|
||||
});
|
||||
counter_features += 1;
|
||||
}
|
||||
}
|
||||
if (headers.indexOf(latColumn) === -1) {
|
||||
console.log('!!! no latColumn', row);
|
||||
counter_missing_lat += 1;
|
||||
}
|
||||
if (headers.indexOf(lonColumn) === -1) {
|
||||
console.log('!!! no lonColumn', row);
|
||||
counter_missing_lon += 1;
|
||||
}
|
||||
}
|
||||
})
|
||||
.on('end', () => {
|
||||
geoJSON = {
|
||||
type: 'FeatureCollection',
|
||||
features,
|
||||
};
|
||||
fs.writeFileSync(`${dir}/${file}.geojson`, JSON.stringify(geoJSON, null, 2));
|
||||
console.log(`GeoJSON créé avec succès : ${file}.geojson`);
|
||||
console.log(`geoJSON lines: ${counter}`);
|
||||
console.log(`geoJSON lines missing lat: ${counter_missing_lat}`);
|
||||
console.log(`geoJSON lines missing lon: ${counter_missing_lon}`);
|
||||
console.log(`geoJSON lines features: ${counter_features}`);
|
||||
});
|
||||
|
||||
return geoJSON;
|
||||
export interface CSVConversionOptions {
|
||||
dir: string;
|
||||
file: string;
|
||||
latColumn?: string;
|
||||
lonColumn?: string;
|
||||
hasHeaders?: boolean;
|
||||
}
|
||||
|
||||
function checkFile(args: Options) {
|
||||
function parseCoordinates(value: string): { lat: number; lon: number } | null {
|
||||
if (!value) return null;
|
||||
|
||||
const parts = value.split(';');
|
||||
if (parts.length >= 2) {
|
||||
const lat = parseFloat(parts[0]);
|
||||
const lon = parseFloat(parts[1]);
|
||||
if (!isNaN(lat) && !isNaN(lon)) {
|
||||
return { lat, lon };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function csvToGeoJSON(options: CSVConversionOptions): FeatureCollection<Point> {
|
||||
const { dir, file, latColumn, lonColumn, hasHeaders } = options;
|
||||
const filePath = path.join(dir, file);
|
||||
|
||||
if (!fs.existsSync(filePath)) {
|
||||
console.error(`Le fichier ${filePath} n'existe pas`);
|
||||
return { type: 'FeatureCollection', features: [] };
|
||||
}
|
||||
|
||||
const csvContent = fs.readFileSync(filePath, 'utf-8');
|
||||
const lines = csvContent.split('\n');
|
||||
const features: Feature<Point>[] = [];
|
||||
|
||||
const startIndex = hasHeaders ? 1 : 0;
|
||||
const headers = hasHeaders ? lines[0].split(';') : null;
|
||||
|
||||
for (let i = startIndex; i < lines.length; i++) {
|
||||
const line = lines[i].trim();
|
||||
if (!line) continue;
|
||||
|
||||
const values = line.split(';');
|
||||
const row: { [key: string]: any } = {};
|
||||
|
||||
if (headers) {
|
||||
headers.forEach((header, index) => {
|
||||
row[header] = values[index];
|
||||
});
|
||||
} else {
|
||||
values.forEach((value, index) => {
|
||||
row[`column${index}`] = value;
|
||||
});
|
||||
}
|
||||
|
||||
let coordinates: { lat: number; lon: number } | null = null;
|
||||
if (latColumn && lonColumn) {
|
||||
const lat = parseFloat(row[latColumn]);
|
||||
const lon = parseFloat(row[lonColumn]);
|
||||
if (!isNaN(lat) && !isNaN(lon)) {
|
||||
coordinates = { lat, lon };
|
||||
}
|
||||
} else if (row['geo_point_2d']) {
|
||||
coordinates = parseCoordinates(row['geo_point_2d']);
|
||||
}
|
||||
|
||||
if (coordinates) {
|
||||
features.push({
|
||||
type: 'Feature',
|
||||
geometry: {
|
||||
type: 'Point',
|
||||
coordinates: [coordinates.lon, coordinates.lat]
|
||||
},
|
||||
properties: row
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
type: 'FeatureCollection',
|
||||
features
|
||||
};
|
||||
}
|
||||
|
||||
function checkFile(args: CSVConversionOptions) {
|
||||
let filePath = path.join(args.dir, args.file);
|
||||
let lineCount = 0;
|
||||
|
||||
|
@ -127,7 +114,7 @@ function checkFile(args: Options) {
|
|||
|
||||
|
||||
}
|
||||
function countGeoJSONFeatures(args: Options) {
|
||||
function countGeoJSONFeatures(args: CSVConversionOptions) {
|
||||
const filePath = path.join(args.dir, `${args.file}.geojson`);
|
||||
|
||||
// Vérifier si le fichier GeoJSON existe
|
||||
|
|
0
etalab_data/plaques_commémoratives/op_uk_stats.txt
Normal file
0
etalab_data/plaques_commémoratives/op_uk_stats.txt
Normal file
245
etalab_data/plaques_commémoratives/stats.txt
Normal file
245
etalab_data/plaques_commémoratives/stats.txt
Normal file
|
@ -0,0 +1,245 @@
|
|||
|
||||
Fichier GeoJSON chargé: etalab_data/plaques_commémoratives/plaques_commemoratives.geojson
|
||||
Nombre de features: 2560
|
||||
|
||||
=== Statistiques des valeurs par propriété ===
|
||||
|
||||
Propriété: y_4326
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 2296
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"48.87378": 8 occurrences (0%)
|
||||
"48.891667": 8 occurrences (0%)
|
||||
"48.892304": 6 occurrences (0%)
|
||||
"48.8871": 5 occurrences (0%)
|
||||
"null": 5 occurrences (0%)
|
||||
... et 2291 autres valeurs
|
||||
|
||||
Propriété: x_4326
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 2351
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"2.295055": 8 occurrences (0%)
|
||||
"2.344681": 6 occurrences (0%)
|
||||
"null": 5 occurrences (0%)
|
||||
"2.333672": 4 occurrences (0%)
|
||||
"2.343397": 4 occurrences (0%)
|
||||
... et 2346 autres valeurs
|
||||
|
||||
Propriété: index_plaque
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 2352
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"1": 194 occurrences (8%)
|
||||
"18052": 3 occurrences (0%)
|
||||
"18062": 2 occurrences (0%)
|
||||
"18063": 2 occurrences (0%)
|
||||
"18065": 2 occurrences (0%)
|
||||
... et 2347 autres valeurs
|
||||
|
||||
Propriété: ardt
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 20
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"6": 280 occurrences (11%)
|
||||
"7": 209 occurrences (8%)
|
||||
"5": 195 occurrences (8%)
|
||||
"4": 189 occurrences (7%)
|
||||
"16": 171 occurrences (7%)
|
||||
... et 15 autres valeurs
|
||||
|
||||
Propriété: adresse
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 2310
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"MÉMORIAL DE LA CONCORDE, JARDIN DES TUILERIES, ...": 10 occurrences (0%)
|
||||
"PLACE CHARLES-DE-GAULLE": 9 occurrences (0%)
|
||||
"QUAI DE LA SEINE": 8 occurrences (0%)
|
||||
"1 PLACE JULES JOFFRIN": 5 occurrences (0%)
|
||||
"RUE BONAPARTE/PLACE SAINT-SULPLICE": 4 occurrences (0%)
|
||||
... et 2305 autres valeurs
|
||||
|
||||
Propriété: emplacement
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 945
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"RDC": 391 occurrences (15%)
|
||||
"null": 279 occurrences (11%)
|
||||
"REZ-DE-CHAUSSEE": 247 occurrences (10%)
|
||||
"PREMIER ETAGE": 65 occurrences (3%)
|
||||
"1ER ETAGE": 44 occurrences (2%)
|
||||
... et 940 autres valeurs
|
||||
|
||||
Propriété: retranscription
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 2519
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"A LA MEMOIRE DES ENFANTS, ELEVES DE CETTE ECOLE...": 11 occurrences (0%)
|
||||
"(médaillon) : 24 AOÛT 1944 / COLONNE DRONNE / L...": 8 occurrences (0%)
|
||||
"A LA MÉMOIRE DES ELEVES DE CETTE ECOLE/DEPORTES...": 3 occurrences (0%)
|
||||
"A LA MÉMOIRE DES ÉLÈVES DE CETTE ÉCOLE / DÉPORT...": 3 occurrences (0%)
|
||||
"A LA MEMOIRE DES ELEVES DE CETTE ECOLE/ DEPORTE...": 3 occurrences (0%)
|
||||
... et 2514 autres valeurs
|
||||
|
||||
Propriété: materiau
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 200
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"marbre": 552 occurrences (22%)
|
||||
"pierre": 446 occurrences (17%)
|
||||
"pierre blanche": 201 occurrences (8%)
|
||||
"Pierre": 180 occurrences (7%)
|
||||
"comblanchien": 176 occurrences (7%)
|
||||
... et 195 autres valeurs
|
||||
|
||||
Propriété: taille
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 399
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"null": 661 occurrences (26%)
|
||||
"30x40": 96 occurrences (4%)
|
||||
"60x40": 64 occurrences (3%)
|
||||
"30x50": 54 occurrences (2%)
|
||||
"20x30": 50 occurrences (2%)
|
||||
... et 394 autres valeurs
|
||||
|
||||
Propriété: titre
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 2348
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"null": 25 occurrences (1%)
|
||||
"Colonne Dronne": 10 occurrences (0%)
|
||||
"Rafle du billet vert": 6 occurrences (0%)
|
||||
"Libération de Paris": 6 occurrences (0%)
|
||||
"Molière": 5 occurrences (0%)
|
||||
... et 2343 autres valeurs
|
||||
|
||||
Propriété: siecle
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 24
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"20": 1436 occurrences (56%)
|
||||
"19-20": 428 occurrences (17%)
|
||||
"19": 267 occurrences (10%)
|
||||
"18": 104 occurrences (4%)
|
||||
"17": 73 occurrences (3%)
|
||||
... et 19 autres valeurs
|
||||
|
||||
Propriété: periode_1
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 20
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"null": 1139 occurrences (44%)
|
||||
"Seconde Guerre mondiale": 1071 occurrences (42%)
|
||||
"Ancien Régime": 72 occurrences (3%)
|
||||
"Première Guerre mondiale": 54 occurrences (2%)
|
||||
"Révolution, Consulat et Empire": 53 occurrences (2%)
|
||||
... et 15 autres valeurs
|
||||
|
||||
Propriété: periode_2
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 11
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"null": 2156 occurrences (84%)
|
||||
"Libération de Paris": 358 occurrences (14%)
|
||||
"Seconde Guerre mondiale": 10 occurrences (0%)
|
||||
"Troisième République": 9 occurrences (0%)
|
||||
"Cinquième République": 8 occurrences (0%)
|
||||
... et 6 autres valeurs
|
||||
|
||||
Propriété: objet_1
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 31
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"les Résistants": 551 occurrences (22%)
|
||||
"les écrivains et intellectuels": 384 occurrences (15%)
|
||||
"lieux, édifices et vestiges": 336 occurrences (13%)
|
||||
"les victimes civiles de guerre": 307 occurrences (12%)
|
||||
"les artistes (beaux-arts)": 152 occurrences (6%)
|
||||
... et 26 autres valeurs
|
||||
|
||||
Propriété: objet_2
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 31
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"null": 1661 occurrences (65%)
|
||||
"les Morts pour la France": 368 occurrences (14%)
|
||||
"les Résistants": 119 occurrences (5%)
|
||||
"les politiques": 71 occurrences (3%)
|
||||
"les écrivains et intellectuels": 56 occurrences (2%)
|
||||
... et 26 autres valeurs
|
||||
|
||||
Propriété: genre
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 4
|
||||
- Toutes les valeurs:
|
||||
"YY": 1659 occurrences (65%)
|
||||
"null": 635 occurrences (25%)
|
||||
"XX": 183 occurrences (7%)
|
||||
"XY": 83 occurrences (3%)
|
||||
|
||||
Propriété: personnalite
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 1771
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"null": 644 occurrences (25%)
|
||||
"Molière": 5 occurrences (0%)
|
||||
"Jeanne d'Arc": 4 occurrences (0%)
|
||||
"George Sand": 3 occurrences (0%)
|
||||
"Pierre Brossolette": 3 occurrences (0%)
|
||||
... et 1766 autres valeurs
|
||||
|
||||
Propriété: pays
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 90
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"France": 2172 occurrences (85%)
|
||||
"null": 35 occurrences (1%)
|
||||
"France, Espagne": 32 occurrences (1%)
|
||||
"France, Pologne": 30 occurrences (1%)
|
||||
"France, Etats-Unis d'Amérique": 28 occurrences (1%)
|
||||
... et 85 autres valeurs
|
||||
|
||||
Propriété: date_arret
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 185
|
||||
- Top 5 des valeurs les plus fréquentes:
|
||||
"null": 2298 occurrences (90%)
|
||||
"octobre 2021": 9 occurrences (0%)
|
||||
"novembre 2021": 8 occurrences (0%)
|
||||
"octobre 2020": 8 occurrences (0%)
|
||||
"novembre 2020": 7 occurrences (0%)
|
||||
... et 180 autres valeurs
|
||||
|
||||
Propriété: geo_point_2d
|
||||
- Total d'occurrences: 2560
|
||||
- Valeurs uniques: 1
|
||||
- Toutes les valeurs:
|
||||
"[object Object]": 2560 occurrences (100%)
|
||||
|
||||
=== Fin des statistiques ===
|
||||
|
||||
Suggestions de mapping OSM:
|
||||
{
|
||||
"y_4326": "",
|
||||
"x_4326": "",
|
||||
"index_plaque": "",
|
||||
"ardt": "",
|
||||
"adresse": "addr:full",
|
||||
"emplacement": "",
|
||||
"retranscription": "",
|
||||
"materiau": "",
|
||||
"taille": "",
|
||||
"titre": "",
|
||||
"siecle": "",
|
||||
"periode_1": "",
|
||||
"periode_2": "",
|
||||
"objet_1": "",
|
||||
"objet_2": "",
|
||||
"genre": "",
|
||||
"personnalite": "",
|
||||
"pays": "",
|
||||
"date_arret": "",
|
||||
"geo_point_2d": ""
|
||||
}
|
||||
Nombre de propriétés uniques: 20
|
12
makefile
12
makefile
|
@ -32,9 +32,19 @@ osmose_irve:
|
|||
#geojson2osm "etalab_data/irve_bornes_recharge/osmose-item-irve-8411-intégrables.json" > "osm_output/osmose-item-irve-8411-intégrables.osm"
|
||||
geojson2osm "output/my_converted_data_set__irve-latest-osmose.json" > "osm_output/my_converted_data_set__irve-latest-osmose.osm"
|
||||
|
||||
# panneaux de signalisation routière extraits de panoramax
|
||||
panneaux:
|
||||
npx ts-node unzip_csv.ts -u https://www.data.gouv.fr/fr/datasets/r/16a20f4e-0c06-40ff-adda-54f214099e5f -o panneaux_limite_de_vitesse_fr_panoramax_detections.csv -z
|
||||
npx ts-node csv_to_geojson.ts -d etalab_data/panneaux -f panneaux_limite_de_vitesse_fr_panoramax_detections.csv --latColumn 'GPSLatitude' --lonColumn 'GPSLongitude' -h
|
||||
ts-node convert_to_osm_tags.ts --source etalab_data/panneaux/panneaux_limite_de_vitesse_fr_panoramax_detections.csv.geojson --output-file=panneaux_maxspeed_panoramax.json --engine-config=MappingPanneauxMaxSpeed
|
||||
|
||||
#plaques commémoratives de Paris
|
||||
plaques:
|
||||
wget https://opendata.paris.fr/api/explore/v2.1/catalog/datasets/plaques_commemoratives/exports/geojson?lang=fr&timezone=Europe%2FBerlin -O "pcp_latest.json"
|
||||
mv pcp_latest.json etalab_data/plaques_commemoratives/pcp_latest.geojson
|
||||
|
||||
npx ts-node csv_to_geojson.ts -d etalab_data/plaques -f plaques_commemoratives_paris.csv --latColumn 'y_4326' --lonColumn 'x_4326' -h
|
||||
ts-node convert_to_osm_tags.ts --source etalab_data/plaques/plaques_commemoratives_paris.csv.geojson --output-file=plaques_commemoratives_paris.json --engine-config=MappingPlaquesCommémorativesParis
|
||||
|
||||
tests:
|
||||
npm test
|
||||
npm test
|
||||
|
|
|
@ -24,7 +24,7 @@ const MappingIRVE: MappingConfigType = {
|
|||
enable_properties_filter: true,
|
||||
// filter_points_older_than_year: 2024,
|
||||
// filter_points_by_year_property: 'date_mise_en_service',
|
||||
// filter_points_lesser_than_NkW: 50 // ne pas sortir les points qui ont moins de ce nombre de puissance nominale
|
||||
// filter_points_lesser_than_NkW: 50 // ne pas sortir les points qui ont moins de ce nombre de puissance nominale
|
||||
// add only geojson points who are found having this regex in the zipcode properties
|
||||
// properties: {
|
||||
// consolidated_code_postal: '^[76|27]'
|
||||
|
@ -36,7 +36,10 @@ const MappingIRVE: MappingConfigType = {
|
|||
add_not_mapped_tags_too: false,
|
||||
boolean_keys: [
|
||||
"prise_type_ef",
|
||||
"prise_type_e",
|
||||
"prise_type2",
|
||||
"prise_type_2",
|
||||
"prise_type_3",
|
||||
"prise_type_combo_ccs",
|
||||
"prise_type_chademo",
|
||||
"gratuit",
|
||||
|
|
86
mappings/converters/configPlaquesCommémorativesParis.ts
Normal file
86
mappings/converters/configPlaquesCommémorativesParis.ts
Normal file
|
@ -0,0 +1,86 @@
|
|||
/**
|
||||
* Plaques commémoratives de Paris
|
||||
*
|
||||
* npx ts-node convert_to_osm_tags.ts --engine-config=MappingPlaquesCommémorativesParis --source=./etalab_data/plaques_commémoratives/plaques_commemoratives.geojson
|
||||
*/
|
||||
import MappingConfigType from "../mapping-config.type";
|
||||
|
||||
const MappingPlaquesCommémorativesParis: MappingConfigType = {
|
||||
config_name: "MappingPlaquesCommémorativesParis",
|
||||
config_author: "tykayn",
|
||||
default_properties_of_point: {
|
||||
// Ajoutez ici les propriétés par défaut pour vos points
|
||||
"memorial": "plaque",
|
||||
"historic": "memorial",
|
||||
},
|
||||
source: {
|
||||
geojson_path: '',
|
||||
url: ''
|
||||
},
|
||||
filters: {
|
||||
// exclude_point_if_tag_not_empty: ['id_osm'], // exclure les points ayant déjà un id_osm pour éviter les doublons
|
||||
// offset: 10
|
||||
|
||||
},
|
||||
add_not_mapped_tags_too: false,
|
||||
boolean_keys: [],
|
||||
tags_to_ignore_if_value_is: ['Non renseigne', 'null'],
|
||||
tags: {
|
||||
|
||||
"index_plaque": "ref:FR:Paris:plaques",
|
||||
// "retranscription": "inscription",
|
||||
"materiau": {
|
||||
key_converted: "material",
|
||||
// remove_original_key: true,
|
||||
conditional_values: {
|
||||
"pierre": {
|
||||
value_converted: "stone"
|
||||
},
|
||||
"pierre blanche": {
|
||||
value_converted: "stone"
|
||||
},
|
||||
"pierre beige": {
|
||||
value_converted: "stone"
|
||||
},
|
||||
"marbre": {
|
||||
value_converted: "marbre"
|
||||
},
|
||||
"marbre fonc\u00e9": {
|
||||
value_converted: "marbre"
|
||||
},
|
||||
"marbre clair": {
|
||||
value_converted: "marbre"
|
||||
},
|
||||
"cuivre": {
|
||||
value_converted: "copper"
|
||||
},
|
||||
"métal": {
|
||||
value_converted: "metal"
|
||||
},
|
||||
"plexiglas transparent": {
|
||||
value_converted: "glass"
|
||||
},
|
||||
"plexiglas opaque": {
|
||||
value_converted: "glass"
|
||||
},
|
||||
"bois": {
|
||||
value_converted: "wood"
|
||||
},
|
||||
"verre": {
|
||||
value_converted: "glass"
|
||||
},
|
||||
"céramique": {
|
||||
value_converted: "ceramic"
|
||||
},
|
||||
|
||||
}
|
||||
},
|
||||
// "titre": "name",
|
||||
// "personnalite": "subject",
|
||||
// -----------------------------------------
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
export default MappingPlaquesCommémorativesParis;
|
|
@ -175,8 +175,7 @@ export default class MappingEngine {
|
|||
}
|
||||
|
||||
|
||||
debugLog('mapElementFromConf: ============= keys mappingKeys:', this.mapping_config.tags.length, mappingKeys.length)
|
||||
debugLog('mapElementFromConf: ============= keys featurePointPropertiesKeys :', featurePoint.properties.length, featurePointPropertiesKeys.length)
|
||||
// console.log('mapElementFromConf: ============= keys mappingKeys:', featurePointPropertiesKeys.length, mappingKeys)
|
||||
|
||||
let newProperties = { ...this.mapping_config.default_properties_of_point }
|
||||
|
||||
|
@ -187,15 +186,16 @@ export default class MappingEngine {
|
|||
basePoint.geometry = featurePoint.geometry
|
||||
basePoint.properties = { ...this.mapping_config.default_properties_of_point }
|
||||
|
||||
|
||||
// apply new properties if found in mapping config
|
||||
featurePointPropertiesKeys.forEach(pointKeyName => {
|
||||
|
||||
debugLog('mapElementFromConf: convert', pointKeyName)
|
||||
debugLog('mapElementFromConf: mapping keys:', mappingKeys)
|
||||
// if (featurePointPropertiesKeys.indexOf(pointKeyName) !== -1) {
|
||||
|
||||
this.convertProperty({
|
||||
pointKeyName, mappingKeys, featurePoint, newProperties
|
||||
})
|
||||
// }
|
||||
|
||||
|
||||
})
|
||||
|
@ -270,7 +270,7 @@ export default class MappingEngine {
|
|||
/**
|
||||
* only use existing keys
|
||||
*/
|
||||
debugLog("only use existing keys,", pointKeyName)
|
||||
// console.log("only use existing keys,", pointKeyName)
|
||||
if (mappingKeys.indexOf(pointKeyName) !== -1) {
|
||||
let valueConvertedFromMapping = featurePoint.properties[pointKeyName]
|
||||
let keyConvertedFromMapping = mappingKeys[mappingKeys.indexOf(pointKeyName)]
|
||||
|
@ -282,6 +282,7 @@ export default class MappingEngine {
|
|||
let convertedValue = originalValue
|
||||
|
||||
let typeOfConfigForKey = typeof mappingConfigOfTag
|
||||
// console.log('typeOfConfigForKey', typeOfConfigForKey)
|
||||
let isStringValue = typeOfConfigForKey === 'string'
|
||||
let isConfigMappingObject = typeOfConfigForKey === 'object'
|
||||
|
||||
|
@ -321,7 +322,7 @@ export default class MappingEngine {
|
|||
if (isConfigMappingObject) {
|
||||
|
||||
|
||||
debugLog('convertProperty: is config object', configObject)
|
||||
// console.log('convertProperty: is config object', configObject)
|
||||
let newKey: any = '' + pointKeyName
|
||||
|
||||
|
||||
|
@ -422,11 +423,13 @@ export default class MappingEngine {
|
|||
debugLog('truncate_enums_to_limit => ', convertedValue)
|
||||
}
|
||||
|
||||
let conditionnalConfig: any = ''
|
||||
/**
|
||||
* config pour une clé
|
||||
* nous pouvons renseigner une string ou un objet décrivant les transformations à réaliser
|
||||
*/
|
||||
if (configObject.conditional_values) {
|
||||
// console.log('configObject.conditional_values', configObject.conditional_values)
|
||||
|
||||
|
||||
// convert numbers from json to string to compare them correctly
|
||||
|
@ -434,15 +437,8 @@ export default class MappingEngine {
|
|||
|
||||
let keysConditionnalValues: any = Object.keys(configObject.conditional_values)
|
||||
let isFoundValue = keysConditionnalValues.indexOf(originalValue)
|
||||
let conditionnalConfig: any = configObject.conditional_values[keysConditionnalValues[isFoundValue]]
|
||||
debugLog('convertProperty: conditional_values__________',
|
||||
configObject.conditional_values)
|
||||
conditionnalConfig = configObject.conditional_values[keysConditionnalValues[isFoundValue]]
|
||||
|
||||
debugLog('isFoundValue', isFoundValue, originalValue)
|
||||
debugLog('keysConditionnalValues', keysConditionnalValues)
|
||||
|
||||
debugLog('-----++++++++ originalValue', originalValue)
|
||||
debugLog('----------- isFoundValue', isFoundValue)
|
||||
|
||||
if (!remove_original_key) {
|
||||
|
||||
|
@ -481,9 +477,11 @@ export default class MappingEngine {
|
|||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
// console.log('convertedValue =>', convertedValue)
|
||||
if (conditionnalConfig?.tags_to_add) {
|
||||
debugLog('on ajoute des tags', conditionnalConfig.tags_to_add)
|
||||
// on peut définir un ensemble de tags à rajouter
|
||||
|
@ -496,7 +494,10 @@ export default class MappingEngine {
|
|||
})
|
||||
}
|
||||
|
||||
} else {
|
||||
debugLog('no conditional values', configObject)
|
||||
}
|
||||
// console.log('conditionnalConfig', conditionnalConfig, convertedValue)
|
||||
|
||||
|
||||
debugLog('convertProperty: convertedValue ==========> {', newKey, ':', convertedValue, '}')
|
||||
|
@ -511,7 +512,12 @@ export default class MappingEngine {
|
|||
}
|
||||
|
||||
debugLog('remove_original_key && newKey && convertedValue && hasKeyIgnoreThisData', remove_original_key, newKey, convertedValue, hasKeyIgnoreThisData)
|
||||
if (!remove_original_key && newKey && convertedValue && !hasKeyIgnoreThisData
|
||||
|
||||
// console.log('newKey && convertedValue && !hasKeyIgnoreThisData', newKey && convertedValue && !hasKeyIgnoreThisData, newKey, convertedValue, !hasKeyIgnoreThisData)
|
||||
if (remove_original_key) {
|
||||
delete newProperties[pointKeyName];
|
||||
}
|
||||
if (newKey && convertedValue && !hasKeyIgnoreThisData
|
||||
) {
|
||||
debugLog('convertedValue', convertedValue)
|
||||
|
||||
|
@ -520,13 +526,16 @@ export default class MappingEngine {
|
|||
}
|
||||
|
||||
}
|
||||
else {
|
||||
debugLog('!!!!!! not isConfigMappingObject: ', isConfigMappingObject)
|
||||
}
|
||||
} else {
|
||||
debugLog('!!!!!! property not found in mappingKeys: ', pointKeyName)
|
||||
}
|
||||
}
|
||||
|
||||
debugLog('newProperties', newProperties)
|
||||
|
||||
// console.log('pointKeyName', pointKeyName)
|
||||
return newProperties;
|
||||
}
|
||||
|
||||
|
|
|
@ -14,7 +14,7 @@ const prefix_phone_fr_only = true
|
|||
function debugLog(...args: any[]) {
|
||||
if (show_debug) {
|
||||
console.log('### debug: ',)
|
||||
args.map((elem: any) => console.log(' ', elem))
|
||||
args.map((elem: any) => console.log(' - ', elem))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -28,9 +28,15 @@ describe('mapping properties with rich mapping engine', () => {
|
|||
})
|
||||
test('maps simple key to key, and keep the same value', () => {
|
||||
let Mapping_engine = new mapping_engine(mappingSame)
|
||||
let featurePoint = {
|
||||
type: 'Feature',
|
||||
properties: {
|
||||
equal: "same value"
|
||||
}
|
||||
}
|
||||
let newProperties = Mapping_engine.convertProperty('equal',
|
||||
Object.keys(mappingSame.tags),
|
||||
feature_to_test,
|
||||
featurePoint,
|
||||
mappingSame.default_properties_of_point)
|
||||
|
||||
expect(newProperties).toStrictEqual({
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue