réinit

etalab_data/gynadco/gather_html.py

@@ -0,0 +1,29 @@
+import os
+import json
+
+# Liste des fichiers JSON à traiter
+json_files = [f"out_250_{i}.json" for i in range(1, 16)]
+
+# Ouvre le fichier list.html en écriture
+with open("list.html", "w", encoding="utf-8") as f_list:
+
+    # Pour chaque fichier JSON
+    for json_file in json_files:
+
+        # Vérifie si le fichier existe
+        if os.path.exists(json_file):
+
+            # Ouvre le fichier JSON en lecture
+            with open(json_file, "r", encoding="utf-8") as f_json:
+
+                # Charge le contenu JSON
+                data = json.load(f_json)
+
+                # Récupère la propriété "html"
+                html_content = data["html"]
+
+                # Écrit le contenu HTML dans le fichier list.html
+                f_list.write(html_content)
+
+        else:
+            print(f"Le fichier {json_file} n'existe pas.")

etalab_data/gynadco/get.sh

@@ -0,0 +1 @@
+curl 'https://gynandco.wordpress.com/?infinity=scrolling' --compressed -X POST -H 'User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:131.0) Gecko/20100101 Firefox/131.0' -H 'Accept: */*' -H 'Accept-Language: fr,en-US;q=0.7,en;q=0.3' -H 'Accept-Encoding: gzip, deflate, br, zstd' -H 'X-Requested-With: XMLHttpRequest' -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' -H 'Origin: https://gynandco.wordpress.com' -H 'Sec-GPC: 1' -H 'Alt-Used: gynandco.wordpress.com' -H 'Connection: keep-alive' -H 'Referer: https://gynandco.wordpress.com/trouver-un-e-soignant-e/liste-des-soignantes/page/1/' -H 'Cookie: ccpa_applies=true; usprivacy=1YNN; euconsent-v2=CQG1_8AQG1_8AECACAFRBMEgAAAAAELAAKiQAAAMnABAECoAAKAAICCAAAIAAAAgCAAgAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgACAAAIAAAAAAAAAAAAEAAAA' -H 'Sec-Fetch-Dest: empty' -H 'Sec-Fetch-Mode: cors' -H 'Sec-Fetch-Site: same-origin' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' -H 'TE: trailers' --data-raw 'action=infinite_scroll&page=10&currentday=17.09.24&order=DESC&scripts[0]=jetpack-mu-wpcom-settings&scripts[1]=rlt-proxy&scripts[2]=media-video-jwt-bridge&scripts[3]=cmp-script-stub&scripts[4]=wpcom-actionbar-placeholder&scripts[5]=grofiles-cards&scripts[6]=wpgroho&scripts[7]=mobile-useragent-info&scripts[8]=wordads-smart&scripts[9]=the-neverending-homepage&scripts[10]=cmp-script&scripts[11]=jetpack-carousel&scripts[12]=tiled-gallery&scripts[13]=carousel-wpcom&styles[0]=wordads-global&styles[1]=free-site-marketing-bar&styles[2]=the-neverending-homepage&styles[3]=wp-block-library&styles[4]=wp-block-library-theme&styles[5]=jetpack-layout-grid&styles[6]=jetpack-ratings&styles[7]=mediaelement&styles[8]=wp-mediaelement&styles[9]=coblocks-frontend&styles[10]=wpcom-core-compat-playlist-styles&styles[11]=wpcom-text-widget-styles&styles[12]=wpcom-bbpress2-staff-css&styles[13]=twentyeleven-block-style&styles[14]=twentyeleven-wpcom-style&styles[15]=reblogging&styles[16]=geo-location-flair&styles[17]=infinity-twentyeleven&styles[18]=h4-global&styles[19]=wp-emoji-styles&styles[20]=videopress-video-style&styles[21]=jetpack-sharing-buttons-style&styles[22]=classic-theme-styles&styles[23]=global-styles&styles[24]=jetpack-global-styles-frontend-style&styles[25]=jetpack-carousel-swiper-css&styles[26]=jetpack-carousel&styles[27]=tiled-gallery&styles[28]=core-block-supports-duotone&query_args[page]=0&query_args[pagename]=trouver-un-e-soignant-e%2Fliste-des-soignantes&query_args[error]=&query_args[m]=&query_args[p]=0&query_args[post_parent]=&query_args[subpost]=&query_args[subpost_id]=&query_args[attachment]=&query_args[attachment_id]=0&query_args[name]=&query_args[page_id]=0&query_args[second]=&query_args[minute]=&query_args[hour]=&query_args[day]=0&query_args[monthnum]=0&query_args[year]=0&query_args[w]=0&query_args[category_name]=&query_args[tag]=&query_args[cat]=&query_args[tag_id]=&query_args[author]=&query_args[author_name]=&query_args[feed]=&query_args[tb]=&query_args[paged]=0&query_args[meta_key]=&query_args[meta_value]=&query_args[preview]=&query_args[s]=&query_args[sentence]=&query_args[title]=&query_args[fields]=&query_args[menu_order]=&query_args[embed]=&query_args[category__in][]=&query_args[category__not_in][]=&query_args[category__and][]=&query_args[post__in][]=&query_args[post__not_in][]=&query_args[post_name__in][]=&query_args[tag__in][]=&query_args[tag__not_in][]=&query_args[tag__and][]=&query_args[tag_slug__in][]=&query_args[tag_slug__and][]=&query_args[post_parent__in][]=&query_args[post_parent__not_in][]=&query_args[author__in][]=&query_args[author__not_in][]=&query_args[search_columns][]=&query_args[lazy_load_term_meta]=false&query_args[posts_per_page]=250&query_args[ignore_sticky_posts]=false&query_args[suppress_filters]=false&query_args[cache_results]=true&query_args[update_post_term_cache]=true&query_args[update_menu_item_cache]=false&query_args[update_post_meta_cache]=true&query_args[post_type]=&query_args[nopaging]=false&query_args[comments_per_page]=50&query_args[no_found_rows]=false&query_args[order]=DESC&query_before=2024-10-21%2022%3A57%3A29&last_post_date=2024-10-15%2018%3A25%3A29'

etalab_data/gynadco/scrap.py

@@ -0,0 +1,231 @@
+import json
+import geopandas as gpd
+from bs4 import BeautifulSoup
+from shapely.geometry import Point
+import re
+import pandas as pd
+
+def extraire_numero_telephone(line):
+    # Extraction des nombres dans la ligne
+    numbers = re.findall(r'\d+', line)
+
+    if numbers:
+        numbers = ''.join(numbers)
+#         print('numbers',numbers)
+    # Vérification si un numéro de téléphone est présent
+    if len(numbers) == 10:
+        # Reconstruction du numéro de téléphone
+        phone_number = ''.join(numbers)
+        return phone_number
+    else:
+        return None
+
+def extraire_code_postal(line):
+    # Extraction du code postal dans la ligne
+    match = re.search(r'\b(\d{5})\b', line)
+    if match:
+        # Extraction des nombres correspondant au code postal
+        code_postal = match.group(1)
+        return code_postal
+    else:
+        return None
+# trouver si la ligne est une adresse en cherchant deux numéros distincts et une virgule
+def extraire_addr_line(line):
+    # Extraction des nombres dans la ligne
+    numbers = re.findall(r'\d+', line)
+
+    # Vérification si un numéro de téléphone est présent
+    if len(numbers) == 2 and ',' in line and len(numbers[1]) == 5:
+        return line
+    else:
+        return None
+
+
+# Charger le fichier HTML
+with open("list.html", "r") as file:
+    html = file.read()
+
+# Analyser le code HTML avec BeautifulSoup
+soup = BeautifulSoup(html, "html.parser")
+
+# Initialiser une liste pour stocker les informations des docteurs
+doctors = []
+# Trouver toutes les balises <article> sur la page
+articles = soup.find_all("article")
+# Parcourir chaque <article> pour extraire les informations des docteurs
+for article in articles:
+    # Récupérer le nom du docteur à partir de la balise <h1>
+#     print(article.find("h1"))
+    name = article.find("h1").find('a').text.strip()
+    url = article.find("h1").find('a').get("href")
+
+    # Récupérer l'adresse du docteur à partir de la balise <em> dans la classe "entry-content"
+    address = ''
+
+    em = article.find("em")
+    if em:
+        address = em.text
+#         print(address)
+
+    # Vérifier si le contenu de l'article contient "Secteur 1"
+    if "Secteur 1" in article.text:
+        sector = "1"
+    elif "Secteur 2" in article.text:
+        sector = "2"
+    else:
+        sector = None
+
+     # Recherche d'un numéro de téléphone dans l'article
+    phone_number = None
+    gender = 'unknown'
+    trans_friendly = ''
+    handles_violence = ''
+    visio_meeting = 'no'
+    pseudo_science = ''
+    premenstrual_syndrome = ''
+    accessible_cabinet = ''
+    tatoo = ''
+    toxico = ''
+    sterilisation = ''
+    abortion = ''
+    endometriosis = ''
+    bigbody = ''
+    poil = ''
+    bi = ''
+    lesbian = ''
+    pma = ''
+    ist = ''
+    pregnancy = ''
+    ivg = ''
+    generaliste = ''
+    gyneco = ''
+    sage_femme = ''
+    auto_prelev = ''
+    mycoses = ''
+    dyspareunie = ''
+    spoken = '' # langues parlées
+    diu = '' # dispositif intra utérin
+    puma= '' # PUMA (ex-CMU)
+    ame= '' # AME (Aide médicale d’état)
+    code_postal = ''
+
+    for line in article.stripped_strings:
+        found = extraire_code_postal(line)
+        if found:
+            code_postal = found
+            address = line
+        found = extraire_numero_telephone(line)
+        if found:
+#             print(found)
+            phone_number = found
+        if 'Rdv en ligne possible' in line:
+            visio_meeting = 'yes'
+        if 'femme soignante' in line:
+            gender = 'women'
+        if 'homme soignante' in line:
+            gender = 'women'
+        if 'Trans friendly' in line:
+            trans_friendly = 'yes'
+        if 'Tattoo' in line:
+            tatoo = 'yes'
+        if 'Sensibilité violence' in line:
+            handles_violence = 'yes'
+        if 'naturelles/alternatives' in line:
+            pseudo_science = 'yes'
+        if 'SPM' in line:
+            premenstrual_syndrome = 'yes'
+        if 'Poilfriendly' in line:
+            poil = 'yes'
+        if 'Bifriendly' in line:
+            bi = 'yes'
+        if 'Stérilisation' in line:
+            sterilisation = 'yes'
+        if 'DIU' in line:
+            diu = 'yes'
+        if 'Accompagnement grossesse' in line:
+            pregnancy = 'yes'
+        if 'suivi des IST' in line:
+            ist = 'yes'
+        if 'IVG' in line:
+            ivg = 'yes'
+        if 'Médecin généraliste' in line:
+            generaliste = 'yes'
+        if 'Sage-femme' in line:
+            sage_femme = 'yes'
+        if 'auto prélèvement' in line:
+            auto_prelev = 'yes'
+        if 'Conseils mycoses' in line:
+            mycoses = 'yes'
+        if 'Lesbiennes friendly' in line:
+            lesbian = 'yes'
+        if 'PMA' in line:
+            pma = 'yes'
+        if 'PUMA' in line:
+            puma = 'yes'
+        if 'AME' in line:
+            ame = 'yes'
+        if 'français, anglais' in line:
+            spoken = 'french;english'
+        if 'espagnol' in line:
+            spoken = ('french;english;spanish')
+        if 'Dyspareunie' in line:
+            dyspareunie = 'yes'
+        if 'Gynécologue' in line:
+            gyneco = 'yes'
+        # chercher une adresse si on en a pas
+        if not address:
+            found = extraire_addr_line(line)
+            if found:
+#                 print(found)
+                address = found
+
+
+    # Ajouter les informations du docteur à la liste
+    doctor = {
+        "name": name,
+        "address": address,
+        "address:code_postal": code_postal,
+        "ref:FR:convention_secteur": sector,
+        "contact:phone": phone_number,
+        "contact:website": url,
+        "gender": gender,
+        "visioconference_meeting": visio_meeting,
+        "handles:gender:trans": trans_friendly,
+        "handles:violence": handles_violence,
+        "handles:premenstrual_syndrome": premenstrual_syndrome,
+        "handles:IST": ist,
+        "accessible_cabinet": accessible_cabinet,
+        "pseudo_science": pseudo_science,
+        "speaks": spoken,
+        "handles:tatoo": tatoo,
+        "handles:toxico": toxico,
+        "handles:sterilisation": sterilisation,
+        "handles:abortion": abortion,
+        "handles:endometriosis": endometriosis,
+        "handles:premenstrual_syndrome": premenstrual_syndrome,
+        "handles:hairy": poil,
+        "handles:bigbody": bigbody,
+        "handles:gender:bi": bi,
+        "handles:diu": diu,
+        "handles:mycoses": mycoses,
+        "handles:pregnancy": pregnancy,
+        "handles:abortion": ivg,
+        "handles:auto_prelevement": auto_prelev,
+        "handles:pma": pma,
+        "handles:dyspareunie": dyspareunie,
+        "handles:gender:lesbian": lesbian,
+        "healcare:generaliste": generaliste,
+        "healcare:sage_femme": sage_femme,
+        "healcare:gynecologist": gyneco,
+
+    }
+    doctors.append(doctor)
+
+# Enregistrer les informations des docteurs au format JSON dans un fichier
+with open("gynandco.json", "w", encoding="utf-8") as f:
+    json.dump(doctors, f, ensure_ascii=False, indent=2)
+
+with open('gynandco.json', 'r') as f:
+    data = json.load(f)
+df = pd.DataFrame(data)
+df.to_csv('gynandco.csv', index=False)