diff --git a/.gitignore b/.gitignore index 712339a..99b19ef 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,9 @@ __pycache__/ *.py[cod] *$py.class .DS_Store -.env \ No newline at end of file +.env +venv +datasources +extractors/**/*.zip +extractors/**/*.7z +extractors/**/*.json diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..31f2e72 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "datasources"] + path = datasources + url = https://github.com/openeventdatabase/datasources.git diff --git a/datasources b/datasources new file mode 160000 index 0000000..4f1410e --- /dev/null +++ b/datasources @@ -0,0 +1 @@ +Subproject commit 4f1410e69cbee6c1c4f8864240379678ad6162cd diff --git a/extractors/bison_fute/bisonfute.py b/extractors/bison_fute/bisonfute.py new file mode 100644 index 0000000..0a8d23e --- /dev/null +++ b/extractors/bison_fute/bisonfute.py @@ -0,0 +1,63 @@ +import requests +import json +import time +from pyproj import Proj, transform + +""" +Scrapping des données de circulation sur bisonfute.fr +export en stream geojson avec 1 événement par ligne +""" + +def charge(x,y,z): + nb = 0 + deeper = False + url = dataurl+("trafic/maintenant/tfs/evenements/%s/%s/%s.json" % (z,x,y)) + + print(f'scraping sur l url : ${url}') + + datareq = requests.get(url=url) + datajson = json.loads(datareq.text) + if not datajson['empty']: + for e in datajson['features']: + if e['properties']['urlImage'][7]=='1': + deeper = True + lon,lat = transform(s_srs,t_srs,e['geometry']['coordinates'][0],e['geometry']['coordinates'][1]) + geometry=dict(type = 'Point', coordinates = [round(lon,6),round(lat,6)]) + quoi = e['properties']['urlImage'][11:-4] + if quoi == 'travaux': + quoi = 'roadwork' + if quoi == 'bouchon': + quoi = 'jam' + e_what = 'traffic.'+quoi + + detailreq = requests.get('http://www.bison-fute.gouv.fr/' + e['properties']['urlcpc']) + detail = json.loads(detailreq.text) + if len(detail)>1: + deeper = True + else: + for d in detail: + print(json.dumps({"geometry":geometry, "properties":{"data":d}})) +# +# if deeper: +# charge(2*x, 2*y, z+1) +# charge((2*x)+1, 2*y, z+1) +# charge(2*x, (2*y)+1, z+1) +# charge((2*x)+1, (2*y)+1, z+1) +# + return nb + +# projections utilisées pour transformation en WGS84 +s_srs = Proj(init='EPSG:2154') +t_srs = Proj(init='EPSG:4326') + +# récupération date courante +datereq = requests.get(url='http://www.bison-fute.gouv.fr/data/iteration/date.json') +datejson = json.loads(datereq.text) +dernier = time.strftime('%Y%m%d-%H%M%S', time.localtime(datejson[0]/1000)) +dataurl = "http://www4.bison-fute.gouv.fr/data/data-%s/" % dernier + +print(f'dataurl : ${dataurl}') + +for x0 in range(0,5): + for y0 in range (0,5): + charge(x0,y0,1) diff --git a/extractors/bison_fute/bisonfute2016.csv b/extractors/bison_fute/bisonfute2016.csv new file mode 100644 index 0000000..9667d13 --- /dev/null +++ b/extractors/bison_fute/bisonfute2016.csv @@ -0,0 +1,71 @@ +date,aller,retour +02/01/16,,1O +03/01/16,,O +06/02/16,O4R, +13/02/16,O4N,4O6O +19/02/16,O, +20/02/16,O4N,3R4R +26/02/16,1O, +27/02/16,1O4O,O4R +05/03/16,4O,1O4O +25/03/16,O1R4R, +26/03/16,O, +28/03/16,,O1R2R +02/04/16,3O4O, +09/04/16,O, +15/04/16,1O2O, +16/04/16,O,O +22/04/16,1O, +23/04/16,1O4O, +24/04/16,,1O +30/04/16,1O, +01/05/16,,O +04/05/16,R, +05/05/16,O, +08/05/16,,R +13/05/16,O1R, +14/05/16,O, +16/05/16,,O +01/07/16,O1R, +02/07/16,O, +08/07/16,O1R, +09/07/16,R, +10/07/16,1O, +13/07/16,O1R, +14/07/16,O, +16/07/16,R, +17/07/16,,O +22/07/16,O1R, +23/07/16,R,O +29/07/16,R,O +30/07/16,N,O6R +31/07/16,O, +05/08/16,O,6O +06/08/16,N,O6N +07/08/16,6O,6O +12/08/16,6O,6O +13/08/16,R,R +14/08/16,6O,5O6O +19/08/16,,O +20/08/16,O6R,R +21/08/16,,O4R +22/08/16,,1O +26/08/16,,O +27/08/16,4O6O,R +28/08/16,,O +29/08/16,,1O +21/10/16,1O, +28/10/16,O1R, +29/10/16,1O, +01/11/16,,O +02/11/16,,1O +10/11/16,O, +11/11/16,1O, +13/11/16,,1O +16/12/16,O, +17/12/16,3O4O, +22/12/16,1O, +23/12/16,O1R, +24/12/16,O4R, +26/12/16,1O4O,1O +02/01/17,,O diff --git a/extractors/bison_fute/bisonfute_previ.py b/extractors/bison_fute/bisonfute_previ.py new file mode 100644 index 0000000..8f2b2f3 --- /dev/null +++ b/extractors/bison_fute/bisonfute_previ.py @@ -0,0 +1,47 @@ +import requests +import json +import time +import csv +import psycopg2 + +pg = psycopg2.connect("dbname=oedb") +db = pg.cursor() + +api = 'http://localhost:8080' + +with open('bisonfute2016.csv') as csvfile: + trafic = csv.DictReader(csvfile, delimiter=',', quotechar='"') + for row in trafic: + start = '20'+row['date'][6:8]+'/'+row['date'][3:5]+'/'+row['date'][0:2]+'T00:00:00CET' + stop = '20'+row['date'][6:8]+'/'+row['date'][3:5]+'/'+row['date'][0:2]+'T23:59:59CET' + for sens in ['aller','retour']: + if row[sens] != '': + if row[sens][0]>'A': + defaut = row[sens][0] + else: + defaut = '' + for zone in range(1,7): + if row[sens].find(str(zone)): + couleur = row[sens][row[sens].find(str(zone))+1] + else: + couleur = defaut + if couleur > 'A': + db.execute('SELECT ST_asgeojson(geom) FROM bison_fute_geo WHERE zone = %s', (str(zone),)) + geo = db.fetchone() + if geo is not None: + what = 'traffic.forecast' + if couleur == 'O': + what = what + '.orange' + if couleur == 'R': + what = what + '.red' + if couleur == 'N': + what = what + '.black' + if sens == 'aller': + what = what + '.out' + if sens == 'retour': + what = what + '.return' + p = dict(type='forecast',what=what, start=start, stop=stop, source='http://www.bison-fute.gouv.fr') + geojson = json.dumps(dict(type='Feature',properties=p, geometry=json.loads(geo[0]))) + r = requests.post(api+'/event', data = geojson, ) + print("%s POST: %s %s" % (r.status_code, r.text,json.dumps(p))) + diff --git a/extractors/bison_fute/init.sh b/extractors/bison_fute/init.sh new file mode 100644 index 0000000..0d7671e --- /dev/null +++ b/extractors/bison_fute/init.sh @@ -0,0 +1,28 @@ + +sudo apt install unzip ogr2ogr + +wget "https://www.data.gouv.fr/api/1/datasets/r/0e117c06-248f-45e5-8945-0e79d9136165" -O communes.shp.zip +unzip -o communes-shp.zip +ogr2ogr -f postgresql PG:"dbname=oedb" communes.shp -nln communes -nlt geometry + +psql oedb -c " +create index communes_insee on communes (insee); +alter table communes drop ogc_fid; +alter table communes drop surf_ha; +" + +wget "https://www.data.gouv.fr/datasets/contours-des-departements-francais-issus-d-openstreetmap/#/resources/eb36371a-761d-44a8-93ec-3d728bec17ce" -O departements-shp.zip +unzip -o departements-shp.zip +ogr2ogr -f postgresql PG:"dbname=oedb" departements-20230101.shp -nln departements -nlt geometry + +psql oedb -c " +create index departements_insee on departements (insee); +alter table departements drop ogc_fid; +alter table departements drop surf_ha; +" +rm departements-* + +# créer la table bison futé et importer les données +sudo -u postgres psql oedb -c "create table bison_fute_zones (dep text, zone text);" +sudo -u postgres psql oedb -c "\copy bison_fute_zones from bisonfute2016.csv with (format csv, header true);" +sudo -u postgres psql oedb -c "create materialized view bison_fute_geo as select zone, st_snaptogrid(st_union(wkb_geometry),0.00001) as geom FROM bison_fute_zones b join departements d on (d.insee=b.dep) group by 1;" diff --git a/extractors/bison_fute/minute.sh b/extractors/bison_fute/minute.sh new file mode 100644 index 0000000..ff9fb57 --- /dev/null +++ b/extractors/bison_fute/minute.sh @@ -0,0 +1,19 @@ +HIERYY=$(date --date="-1 day" +'%Y') +HIERMM=$(date --date="-1 day" +'%m') +HIER=$(date --date="-1 day" +'%Y-%m-%d') + +mkdir -p `date +%Y-%m` +f=`date +%Y-%m`/bison-fute-`date +%Y-%m-%dT%H:%M:00%z`.json +l="last.json" +py bisonfute.py > $f +# ~/.virtualenvs/oedb/bin/python parks.py +if diff $f $l >/dev/null +then + rm $f + touch -h $l +else + ln -fs $f $l +fi + +# archivage des données de la veille +7z a $HIERYY.7z $HIERYY-$HIERMM/*$HIER* && rm $HIERYY-$HIERMM/*$HIER*