mirror of
https://forge.chapril.org/tykayn/wololo
synced 2025-06-20 01:34:42 +02:00
add option --wget to replace file
This commit is contained in:
Tykayn
tykayn
parent
20b9e24eea
commit
5b0271716c
7 changed files with 128 additions and 6 deletions
|
|
@ -12,6 +12,7 @@
|
|||
* --outname=nom_fichier : Alias pour --output-file
|
||||
* --testingConfig : Active le mode test avec la configuration mappingTest
|
||||
*/
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
/**
|
||||
* Mesure de l'utilisation de la RAM
|
||||
|
|
@ -71,7 +72,7 @@ let default_engine_conf_choice: string = 'ConfigIRVE'
|
|||
let engine_conf_choice: string = 'ConfigIRVE'
|
||||
let filterCoordinates = true
|
||||
filterCoordinates = false
|
||||
|
||||
let wget = false
|
||||
let enable_filter_on_department = true
|
||||
enable_filter_on_department = false
|
||||
|
||||
|
|
@ -117,6 +118,37 @@ if (mini_arguments['testingConfig']) {
|
|||
Mapping_engine = new mapping_engine(config.allowed_configs.mappingConfigIRVE)
|
||||
}
|
||||
|
||||
// refresh the data from the source specified in the mapping config
|
||||
if (mini_arguments['wget']) {
|
||||
wget = mini_arguments['wget'];
|
||||
}
|
||||
|
||||
|
||||
|
||||
async function replaceFile(sourceFilePathGeoJson: string, url: string) {
|
||||
const response = await fetch(url)
|
||||
if (!response.ok) {
|
||||
throw new Error(`Erreur lors du téléchargement: ${response.status} ${response.statusText}`)
|
||||
}
|
||||
// afficher la taille du fichier téléchargé
|
||||
const contentLength = response.headers.get('content-length')
|
||||
|
||||
const data = await response.text()
|
||||
if (contentLength) {
|
||||
const sizeInMB = (parseInt(contentLength) / (1024 * 1024)).toFixed(2)
|
||||
console.log(`Taille du fichier téléchargé: ${sizeInMB} Mo`)
|
||||
} else {
|
||||
|
||||
// mesurer la taille des données
|
||||
const sizeInMB = (data.length / (1024 * 1024)).toFixed(2)
|
||||
console.log(`Taille des données: ${sizeInMB} Mo`)
|
||||
}
|
||||
|
||||
fs.writeFileSync(sourceFilePathGeoJson, data)
|
||||
console.log('fichier téléchargé avec succès:', sourceFilePathGeoJson)
|
||||
|
||||
}
|
||||
|
||||
let filterZipCode = new RegExp(`^${filterDepartment}`)
|
||||
let filterZipCodeAdresse = new RegExp(` ${filterDepartment}`)
|
||||
let filteredName = ''
|
||||
|
|
@ -468,8 +500,33 @@ function setMappingConfigFromName(engine_conf_choice: string) {
|
|||
return
|
||||
}
|
||||
}
|
||||
let currentMappingConfig = Mapping_engine.getConfig();
|
||||
convertDataFromSource(sourceFilePathGeoJson, currentMappingConfig, pointCounterMax, boundingBoxCoordinates)
|
||||
let currentMappingConfig = Mapping_engine.getConfig() as MappingConfigType;
|
||||
|
||||
// si on a wget et que le mapping config a une source, on récupère la source définie dans le mapping config
|
||||
if (wget) {
|
||||
// console.log('wget enabled', currentMappingConfig.source)
|
||||
// let geojson_path = currentMappingConfig.source.geojson_path
|
||||
// let filename = sourceFilePathGeoJson.split('/').pop()
|
||||
// télécharger le fichier source depuis l'url du mapping config
|
||||
if (currentMappingConfig.source.geojson_path) {
|
||||
console.log('téléchargement du fichier source depuis', currentMappingConfig.source.geojson_path)
|
||||
console.log('la documentation de la ressource est disponible sur ', currentMappingConfig.source.url)
|
||||
try {
|
||||
|
||||
replaceFile(sourceFilePathGeoJson, currentMappingConfig.source.geojson_path)
|
||||
convertDataFromSource(sourceFilePathGeoJson, currentMappingConfig, pointCounterMax, boundingBoxCoordinates)
|
||||
} catch (error) {
|
||||
console.error('Erreur lors du téléchargement du fichier source:', error)
|
||||
process.exit(1)
|
||||
}
|
||||
} else {
|
||||
console.log('wget enabled, but no geojson_path found in mapping config', currentMappingConfig.source)
|
||||
}
|
||||
} else {
|
||||
|
||||
|
||||
convertDataFromSource(sourceFilePathGeoJson, currentMappingConfig, pointCounterMax, boundingBoxCoordinates)
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,9 @@
|
|||
* plan de conversion des clés du jeu de données vers les tags OSM
|
||||
* détail dans le tableau
|
||||
* https://wiki.openstreetmap.org/wiki/France/data.gouv.fr/Bornes_de_Recharge_pour_V%C3%A9hicules_%C3%89lectriques
|
||||
*
|
||||
* exemple d'exécution de script:
|
||||
* npx ts-node convert_to_osm_tags.ts --source="etalab_data/irve_bornes_recharge/latest.json" --output-file="irve-latest.geojson" --engine-config=mappingConfigIRVE --wget
|
||||
*/
|
||||
import MappingConfigType from "../mapping-config.type";
|
||||
|
||||
|
|
@ -12,8 +15,10 @@ const MappingIRVE: MappingConfigType = {
|
|||
'amenity': 'charging_station'
|
||||
},
|
||||
source: {
|
||||
geojson_path: "etalab_data/all.json",
|
||||
url: 'https://www.data.gouv.fr/fr/datasets/r/7eee8f09-5d1b-4f48-a304-5e99e8da1e26'
|
||||
geojson_path: "https://www.data.gouv.fr/fr/datasets/r/7eee8f09-5d1b-4f48-a304-5e99e8da1e26",
|
||||
url: 'https://www.data.gouv.fr/fr/datasets/r/7eee8f09-5d1b-4f48-a304-5e99e8da1e26',
|
||||
documentation_url: 'https://transport.data.gouv.fr/datasets/fichier-consolide-des-bornes-de-recharge-pour-vehicules-electriques',
|
||||
overpass_query: '[out:json][timeout:300];area(id:3602202162)->.searchArea;nwr["amenity"="charging_station"](area.searchArea);out+geom;'
|
||||
},
|
||||
/**
|
||||
* select only certain points from the source
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@
|
|||
*
|
||||
* certaines ont une précision "Caméra de sécurité sur borne escamotable"
|
||||
* https://wiki.openstreetmap.org/wiki/Tag:man_made=surveillance
|
||||
* exemple d'exécution de script:
|
||||
* ts-node convert_to_osm_tags.ts --engine-config configSurveillance --outname surveillance_rouen_20250428 --wget
|
||||
*/
|
||||
import MappingConfigType from "../mapping-config.type";
|
||||
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ interface sourceConfig {
|
|||
geojson_path: string; // the relative path to the geojson source file to analyse, from the root of this repository
|
||||
url: string; // URL from where the geojson comes online, on a data platform. This URL should be fetchable to get the most recent data of the concerned dataset to convert.
|
||||
overpass_query?: string; // query to get objects from OSM
|
||||
documentation_url?: string; // documentation on open data website about this dataset
|
||||
}
|
||||
|
||||
export default interface MappingConfigType {
|
||||
|
|
|
|||
|
|
@ -159,7 +159,7 @@ function convertToYesOrNo(originalValue: any) {
|
|||
}
|
||||
// on ne peut pas conclure ce que l'on doit garder avec une liste
|
||||
if (isEnumeration) {
|
||||
this.stats.enumeration_detected++
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
|
|
|
|||
55
package-lock.json
generated
55
package-lock.json
generated
|
|
@ -12,6 +12,7 @@
|
|||
"@types/geojson2osm": "^0.0.3",
|
||||
"csv": "^6.3.1",
|
||||
"geojsontoosm": "^0.0.3",
|
||||
"node-fetch": "^2.7.0",
|
||||
"node-fs": "^0.1.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
@ -22,6 +23,7 @@
|
|||
"@turf/turf": "^7.2.0",
|
||||
"@types/minimist": "^1.2.2",
|
||||
"@types/node": "^20.4.7",
|
||||
"@types/node-fetch": "^2.6.12",
|
||||
"axios": "^1.8.4",
|
||||
"babel-jest": "^29.6.2",
|
||||
"csv-parser": "^3.2.0",
|
||||
|
|
@ -4824,6 +4826,17 @@
|
|||
"integrity": "sha512-dP7f3LdZIysZnmvP3ANJYTSwg+wLLl8p7RqniVlV7j+oXSXAbt9h0WIBFmJy5inWZoX9wZN6eXx+YXd9Rh3RBA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/node-fetch": {
|
||||
"version": "2.6.12",
|
||||
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz",
|
||||
"integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*",
|
||||
"form-data": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/stack-utils": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz",
|
||||
|
|
@ -8387,6 +8400,26 @@
|
|||
"integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/node-fetch": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
||||
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"whatwg-url": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "4.x || >=6.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"encoding": "^0.1.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"encoding": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/node-fs": {
|
||||
"version": "0.1.7",
|
||||
"resolved": "https://registry.npmjs.org/node-fs/-/node-fs-0.1.7.tgz",
|
||||
|
|
@ -9320,6 +9353,12 @@
|
|||
"nodetouch": "bin/nodetouch.js"
|
||||
}
|
||||
},
|
||||
"node_modules/tr46": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/ts-jest": {
|
||||
"version": "29.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.1.1.tgz",
|
||||
|
|
@ -9595,6 +9634,22 @@
|
|||
"makeerror": "1.0.12"
|
||||
}
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/whatwg-url": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
||||
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tr46": "~0.0.3",
|
||||
"webidl-conversions": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/which": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@
|
|||
"@types/geojson2osm": "^0.0.3",
|
||||
"csv": "^6.3.1",
|
||||
"geojsontoosm": "^0.0.3",
|
||||
"node-fetch": "^2.7.0",
|
||||
"node-fs": "^0.1.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
@ -37,6 +38,7 @@
|
|||
"@turf/turf": "^7.2.0",
|
||||
"@types/minimist": "^1.2.2",
|
||||
"@types/node": "^20.4.7",
|
||||
"@types/node-fetch": "^2.6.12",
|
||||
"axios": "^1.8.4",
|
||||
"babel-jest": "^29.6.2",
|
||||
"csv-parser": "^3.2.0",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue