wiki illustrations et team osm fr

wiki_compare/find_pages_unavailable_in_french.py

@@ -27,6 +27,7 @@ import os
 import re
 import random
 import hashlib
+import csv
 from datetime import datetime, timedelta
 import requests
 from bs4 import BeautifulSoup
@@ -41,10 +42,33 @@ logger = logging.getLogger(__name__)
 
 # Constants
 OUTPUT_FILE = "pages_unavailable_in_french.json"
+WIKI_PAGES_CSV = "wiki_pages.csv"
 BASE_URL = "https://wiki.openstreetmap.org/wiki/Category:Pages_unavailable_in_French"
 WIKI_BASE_URL = "https://wiki.openstreetmap.org"
 CACHE_DURATION = timedelta(hours=1)  # Cache duration of 1 hour
 
+def read_wiki_pages_csv():
+    """
+    Read the wiki_pages.csv file and create a mapping of URLs to description_img_url values
+    
+    Returns:
+        dict: Dictionary mapping URLs to description_img_url values
+    """
+    url_to_img_map = {}
+    
+    try:
+        with open(WIKI_PAGES_CSV, 'r', newline='', encoding='utf-8') as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                if 'url' in row and 'description_img_url' in row and row['description_img_url']:
+                    url_to_img_map[row['url']] = row['description_img_url']
+        
+        logger.info(f"Read {len(url_to_img_map)} image URLs from {WIKI_PAGES_CSV}")
+        return url_to_img_map
+    except (IOError, csv.Error) as e:
+        logger.error(f"Error reading {WIKI_PAGES_CSV}: {e}")
+        return {}
+
 def is_cache_fresh():
     """
     Check if the cache file exists and is less than CACHE_DURATION old
@@ -273,6 +297,9 @@ def main():
         logger.info(f"Use --force to update anyway")
         return
     
+    # Read image URLs from wiki_pages.csv
+    url_to_img_map = read_wiki_pages_csv()
+    
     # Scrape pages
     pages = scrape_all_pages()
     
@@ -280,6 +307,11 @@ def main():
         logger.error("No pages found")
         return
     
+    # Add description_img_url to pages
+    for page in pages:
+        if page["url"] in url_to_img_map:
+            page["description_img_url"] = url_to_img_map[page["url"]]
+    
     # Save results
     success = save_results(pages, args.dry_run)
     

wiki_compare/top_keys.json

@@ -18,5 +18,65 @@
   {
     "key": "addr:street",
     "count": 161005721
+  },
+  {
+    "key": "addr:city",
+    "count": 123355107
+  },
+  {
+    "key": "name",
+    "count": 109342549
+  },
+  {
+    "key": "addr:postcode",
+    "count": 107014659
+  },
+  {
+    "key": "natural",
+    "count": 84723029
+  },
+  {
+    "key": "surface",
+    "count": 72309071
+  },
+  {
+    "key": "addr:country",
+    "count": 50567842
+  },
+  {
+    "key": "landuse",
+    "count": 48196369
+  },
+  {
+    "key": "power",
+    "count": 44787307
+  },
+  {
+    "key": "waterway",
+    "count": 37279458
+  },
+  {
+    "key": "building:levels",
+    "count": 36502866
+  },
+  {
+    "key": "amenity",
+    "count": 30994353
+  },
+  {
+    "key": "barrier",
+    "count": 30164354
+  },
+  {
+    "key": "source:date",
+    "count": 29112775
+  },
+  {
+    "key": "service",
+    "count": 28396250
+  },
+  {
+    "key": "addr:state",
+    "count": 25367076
   }
 ]

wiki_compare/wiki_compare.py

@@ -211,6 +211,145 @@ def fetch_wiki_page(key, language='en'):
             
             # Get media details (src and alt text)
             media_details = []
+
+            # Extract description image specifically
+            # Try multiple selectors to find the description image
+            description_img = None
+
+            # Debug: Log the key we're processing
+            logger.info(f"Looking for description image for key '{key}' in {language}")
+
+            # Function to filter out OSM logo and small icons
+            def is_relevant_image(img):
+                src = img.get('src', '')
+                # Skip OSM logo
+                if 'osm_logo' in src:
+                    return False
+                # Skip small icons (usually less than 30px)
+                width = img.get('width')
+                if width and int(width) < 30:
+                    return False
+                height = img.get('height')
+                if height and int(height) < 30:
+                    return False
+                return True
+
+            # Special case for highway key - directly target the image we want
+            if key == 'highway':
+                # Try to find the specific image in figure elements
+                highway_img_elements = content.select('figure.mw-halign-center img')
+                logger.info(f"  Highway specific selector 'figure.mw-halign-center img' found {len(highway_img_elements)} elements")
+
+                # Filter for relevant images
+                relevant_images = [img for img in highway_img_elements if is_relevant_image(img)]
+                logger.info(f"  Found {len(relevant_images)} relevant images for highway")
+
+                if relevant_images:
+                    description_img = relevant_images[0]
+                    logger.info(f"  Using highway-specific image: {description_img.get('src', '')}")
+
+            # If not found with highway-specific selector, try the td.d_image selector
+            if not description_img:
+                description_img_elements = content.select('td.d_image img')
+                logger.info(f"  Selector 'td.d_image img' found {len(description_img_elements)} elements")
+
+                # Filter for relevant images
+                relevant_images = [img for img in description_img_elements if is_relevant_image(img)]
+                logger.info(f"  Found {len(relevant_images)} relevant images in td.d_image")
+
+                if relevant_images:
+                    description_img = relevant_images[0]
+                    logger.info(f"  Using image from 'td.d_image img': {description_img.get('src', '')}")
+
+            # If still not found, try the specific selector for .description img.mw-file-element
+            if not description_img:
+                description_img_elements = content.select('.description img.mw-file-element')
+                logger.info(f"  Selector '.description img.mw-file-element' found {len(description_img_elements)} elements")
+
+                # Filter for relevant images
+                relevant_images = [img for img in description_img_elements if is_relevant_image(img)]
+                logger.info(f"  Found {len(relevant_images)} relevant images in .description")
+
+                if relevant_images:
+                    description_img = relevant_images[0]
+                    logger.info(f"  Using image from '.description img.mw-file-element': {description_img.get('src', '')}")
+
+            # If still not found, try images in figures within the description box
+            if not description_img:
+                description_img_elements = content.select('.description figure img')
+                logger.info(f"  Selector '.description figure img' found {len(description_img_elements)} elements")
+
+                # Filter for relevant images
+                relevant_images = [img for img in description_img_elements if is_relevant_image(img)]
+                logger.info(f"  Found {len(relevant_images)} relevant images in .description figure")
+
+                if relevant_images:
+                    description_img = relevant_images[0]
+                    logger.info(f"  Using image from '.description figure img': {description_img.get('src', '')}")
+
+            # If still not found, try any image in the description box
+            if not description_img:
+                description_img_elements = content.select('.description img')
+                logger.info(f"  Selector '.description img' found {len(description_img_elements)} elements")
+
+                # Filter for relevant images
+                relevant_images = [img for img in description_img_elements if is_relevant_image(img)]
+                logger.info(f"  Found {len(relevant_images)} relevant images in .description general")
+
+                if relevant_images:
+                    description_img = relevant_images[0]
+                    logger.info(f"  Using image from '.description img': {description_img.get('src', '')}")
+
+            # If still not found, try images in the DescriptionBox table
+            if not description_img:
+                description_img_elements = content.select('table.DescriptionBox img')
+                logger.info(f"  Selector 'table.DescriptionBox img' found {len(description_img_elements)} elements")
+
+                # Filter for relevant images
+                relevant_images = [img for img in description_img_elements if is_relevant_image(img)]
+                logger.info(f"  Found {len(relevant_images)} relevant images in DescriptionBox")
+
+                if relevant_images:
+                    description_img = relevant_images[0]
+                    logger.info(f"  Using image from 'table.DescriptionBox img': {description_img.get('src', '')}")
+
+            # If still not found, try images in figure elements anywhere in the content
+            if not description_img:
+                description_img_elements = content.select('figure img')
+                logger.info(f"  Selector 'figure img' found {len(description_img_elements)} elements")
+
+                # Filter for relevant images
+                relevant_images = [img for img in description_img_elements if is_relevant_image(img)]
+                logger.info(f"  Found {len(relevant_images)} relevant images in figure elements")
+
+                if relevant_images:
+                    description_img = relevant_images[0]
+                    logger.info(f"  Using image from 'figure img': {description_img.get('src', '')}")
+
+            # If we still don't have an image, use any image that's not the OSM logo
+            if not description_img:
+                all_images = content.select('img')
+                relevant_images = [img for img in all_images if is_relevant_image(img)]
+                logger.info(f"  Found {len(relevant_images)} relevant images in the entire page")
+
+                if relevant_images:
+                    description_img = relevant_images[0]
+                    logger.info(f"  Using fallback image: {description_img.get('src', '')}")
+
+            # Process the found image
+            description_img_url = None
+            if description_img:
+                src = description_img.get('src', '')
+                if src:
+                    # Make relative URLs absolute
+                    if src.startswith('//'):
+                        src = 'https:' + src
+                    elif src.startswith('/'):
+                        src = 'https://wiki.openstreetmap.org' + src
+
+                    description_img_url = src
+
+            # Process all images
             for img in media_elements:
                 src = img.get('src', '')
                 if src:
@@ -251,7 +390,8 @@ def fetch_wiki_page(key, language='en'):
             'link_details': link_details,
             'media_count': media_count,
             'media_details': media_details,
-            'categories': categories
+            'categories': categories,
+            'description_img_url': description_img_url
         }
     
     except requests.exceptions.RequestException as e:
@@ -692,7 +832,7 @@ def main():
     try:
         with open(WIKI_PAGES_CSV, 'w', newline='', encoding='utf-8') as f:
             # Basic fields for CSV (detailed content will be in JSON only)
-            fieldnames = ['key', 'language', 'url', 'last_modified', 'sections', 'word_count', 'link_count', 'media_count', 'staleness_score']
+            fieldnames = ['key', 'language', 'url', 'last_modified', 'sections', 'word_count', 'link_count', 'media_count', 'staleness_score', 'description_img_url']
             writer = csv.DictWriter(f, fieldnames=fieldnames)
             
             writer.writeheader()

wiki_compare/wiki_pages.csv

@@ -1,11 +1,40 @@
-key,language,url,last_modified,sections,word_count,link_count,media_count,staleness_score
-building,en,https://wiki.openstreetmap.org/wiki/Key:building,2025-06-10,31,3774,627,158,8.91
-building,fr,https://wiki.openstreetmap.org/wiki/FR:Key:building,2025-05-22,25,3181,544,155,8.91
-source,en,https://wiki.openstreetmap.org/wiki/Key:source,2025-08-12,27,2752,314,42,113.06
-source,fr,https://wiki.openstreetmap.org/wiki/FR:Key:source,2024-02-07,23,2593,230,35,113.06
-highway,en,https://wiki.openstreetmap.org/wiki/Key:highway,2025-04-10,30,4126,780,314,20.35
-highway,fr,https://wiki.openstreetmap.org/wiki/FR:Key:highway,2025-01-05,30,4141,695,313,20.35
-addr:housenumber,en,https://wiki.openstreetmap.org/wiki/Key:addr:housenumber,2025-07-24,11,330,97,20,14.01
-addr:housenumber,fr,https://wiki.openstreetmap.org/wiki/FR:Key:addr:housenumber,2025-08-23,15,1653,150,77,14.01
-addr:street,en,https://wiki.openstreetmap.org/wiki/Key:addr:street,2024-10-29,12,602,101,16,66.04
-addr:street,fr,https://wiki.openstreetmap.org/wiki/FR:Key:addr:street,2025-08-23,15,1653,150,77,66.04
+key,language,url,last_modified,sections,word_count,link_count,media_count,staleness_score,description_img_url
+building,en,https://wiki.openstreetmap.org/wiki/Key:building,2025-06-10,31,3774,627,158,8.91,https://wiki.openstreetmap.org/w/images/thumb/6/61/Emptyhouse.jpg/200px-Emptyhouse.jpg
+building,fr,https://wiki.openstreetmap.org/wiki/FR:Key:building,2025-05-22,25,3181,544,155,8.91,https://wiki.openstreetmap.org/w/images/thumb/6/61/Emptyhouse.jpg/200px-Emptyhouse.jpg
+source,en,https://wiki.openstreetmap.org/wiki/Key:source,2025-08-12,27,2752,314,42,113.06,https://wiki.openstreetmap.org/w/images/thumb/7/76/Osm_element_node.svg/30px-Osm_element_node.svg.png
+source,fr,https://wiki.openstreetmap.org/wiki/FR:Key:source,2024-02-07,23,2593,230,35,113.06,https://wiki.openstreetmap.org/w/images/thumb/7/76/Osm_element_node.svg/30px-Osm_element_node.svg.png
+highway,en,https://wiki.openstreetmap.org/wiki/Key:highway,2025-04-10,30,4126,780,314,20.35,https://upload.wikimedia.org/wikipedia/commons/thumb/7/78/Roads_in_Switzerland_%2827965437018%29.jpg/200px-Roads_in_Switzerland_%2827965437018%29.jpg
+highway,fr,https://wiki.openstreetmap.org/wiki/FR:Key:highway,2025-01-05,30,4141,695,313,20.35,https://upload.wikimedia.org/wikipedia/commons/thumb/7/78/Roads_in_Switzerland_%2827965437018%29.jpg/200px-Roads_in_Switzerland_%2827965437018%29.jpg
+addr:housenumber,en,https://wiki.openstreetmap.org/wiki/Key:addr:housenumber,2025-07-24,11,330,97,20,14.01,https://upload.wikimedia.org/wikipedia/commons/thumb/1/16/Ferry_Street%2C_Portaferry_%2809%29%2C_October_2009.JPG/200px-Ferry_Street%2C_Portaferry_%2809%29%2C_October_2009.JPG
+addr:housenumber,fr,https://wiki.openstreetmap.org/wiki/FR:Key:addr:housenumber,2025-08-23,15,1653,150,77,14.01,https://wiki.openstreetmap.org/w/images/thumb/e/e9/Housenumber-karlsruhe-de.png/200px-Housenumber-karlsruhe-de.png
+addr:street,en,https://wiki.openstreetmap.org/wiki/Key:addr:street,2024-10-29,12,602,101,16,66.04,https://upload.wikimedia.org/wikipedia/commons/thumb/6/64/UK_-_London_%2830474933636%29.jpg/200px-UK_-_London_%2830474933636%29.jpg
+addr:street,fr,https://wiki.openstreetmap.org/wiki/FR:Key:addr:street,2025-08-23,15,1653,150,77,66.04,https://wiki.openstreetmap.org/w/images/thumb/e/e9/Housenumber-karlsruhe-de.png/200px-Housenumber-karlsruhe-de.png
+addr:city,en,https://wiki.openstreetmap.org/wiki/Key:addr:city,2025-07-29,15,802,105,17,9.93,https://upload.wikimedia.org/wikipedia/commons/thumb/1/18/Lillerod.jpg/200px-Lillerod.jpg
+addr:city,fr,https://wiki.openstreetmap.org/wiki/FR:Key:addr:city,2025-08-23,15,1653,150,77,9.93,https://wiki.openstreetmap.org/w/images/thumb/e/e9/Housenumber-karlsruhe-de.png/200px-Housenumber-karlsruhe-de.png
+name,en,https://wiki.openstreetmap.org/wiki/Key:name,2025-07-25,17,2196,281,82,42.39,https://upload.wikimedia.org/wikipedia/commons/thumb/6/61/Helena%2C_Montana.jpg/200px-Helena%2C_Montana.jpg
+name,fr,https://wiki.openstreetmap.org/wiki/FR:Key:name,2025-01-16,21,1720,187,60,42.39,https://wiki.openstreetmap.org/w/images/3/37/Strakers.jpg
+addr:postcode,en,https://wiki.openstreetmap.org/wiki/Key:addr:postcode,2024-10-29,14,382,83,11,67.11,https://upload.wikimedia.org/wikipedia/commons/thumb/0/04/Farrer_post_code.jpg/200px-Farrer_post_code.jpg
+addr:postcode,fr,https://wiki.openstreetmap.org/wiki/FR:Key:addr:postcode,2025-08-23,15,1653,150,77,67.11,https://wiki.openstreetmap.org/w/images/thumb/e/e9/Housenumber-karlsruhe-de.png/200px-Housenumber-karlsruhe-de.png
+natural,en,https://wiki.openstreetmap.org/wiki/Key:natural,2025-07-17,17,2070,535,189,22.06,https://upload.wikimedia.org/wikipedia/commons/thumb/0/0e/VocaDi-Nature%2CGeneral.jpeg/200px-VocaDi-Nature%2CGeneral.jpeg
+natural,fr,https://wiki.openstreetmap.org/wiki/FR:Key:natural,2025-04-21,13,1499,455,174,22.06,https://upload.wikimedia.org/wikipedia/commons/thumb/0/0e/VocaDi-Nature%2CGeneral.jpeg/200px-VocaDi-Nature%2CGeneral.jpeg
+surface,en,https://wiki.openstreetmap.org/wiki/Key:surface,2025-08-28,24,3475,591,238,264.64,https://upload.wikimedia.org/wikipedia/commons/thumb/a/a2/Transportation_in_Tanzania_Traffic_problems.JPG/200px-Transportation_in_Tanzania_Traffic_problems.JPG
+surface,fr,https://wiki.openstreetmap.org/wiki/FR:Key:surface,2022-02-22,13,2587,461,232,264.64,https://upload.wikimedia.org/wikipedia/commons/thumb/a/a2/Transportation_in_Tanzania_Traffic_problems.JPG/200px-Transportation_in_Tanzania_Traffic_problems.JPG
+addr:country,en,https://wiki.openstreetmap.org/wiki/Key:addr:country,2024-12-01,9,184,65,11,22.96,https://upload.wikimedia.org/wikipedia/commons/thumb/8/86/Europe_ISO_3166-1.svg/200px-Europe_ISO_3166-1.svg.png
+addr:country,fr,https://wiki.openstreetmap.org/wiki/FR:Key:addr:country,2025-03-25,8,187,65,11,22.96,https://upload.wikimedia.org/wikipedia/commons/thumb/8/86/Europe_ISO_3166-1.svg/200px-Europe_ISO_3166-1.svg.png
+landuse,en,https://wiki.openstreetmap.org/wiki/Key:landuse,2025-03-01,17,2071,446,168,39.41,https://upload.wikimedia.org/wikipedia/commons/thumb/d/d3/Changing_landuse_-_geograph.org.uk_-_1137810.jpg/200px-Changing_landuse_-_geograph.org.uk_-_1137810.jpg
+landuse,fr,https://wiki.openstreetmap.org/wiki/FR:Key:landuse,2024-08-20,19,2053,418,182,39.41,https://upload.wikimedia.org/wikipedia/commons/thumb/d/d3/Changing_landuse_-_geograph.org.uk_-_1137810.jpg/200px-Changing_landuse_-_geograph.org.uk_-_1137810.jpg
+power,en,https://wiki.openstreetmap.org/wiki/Key:power,2025-02-28,20,641,127,21,124.89,https://wiki.openstreetmap.org/w/images/thumb/0/01/Power-tower.JPG/200px-Power-tower.JPG
+power,fr,https://wiki.openstreetmap.org/wiki/FR:Key:power,2023-06-27,14,390,105,25,124.89,https://wiki.openstreetmap.org/w/images/thumb/0/01/Power-tower.JPG/200px-Power-tower.JPG
+waterway,en,https://wiki.openstreetmap.org/wiki/Key:waterway,2025-03-10,21,1830,365,118,77.94,https://wiki.openstreetmap.org/w/images/thumb/f/fe/450px-Marshall-county-indiana-yellow-river.jpg/200px-450px-Marshall-county-indiana-yellow-river.jpg
+waterway,fr,https://wiki.openstreetmap.org/wiki/FR:Key:waterway,2024-03-08,18,1291,272,113,77.94,https://wiki.openstreetmap.org/w/images/thumb/f/fe/450px-Marshall-county-indiana-yellow-river.jpg/200px-450px-Marshall-county-indiana-yellow-river.jpg
+building:levels,en,https://wiki.openstreetmap.org/wiki/Key:building:levels,2025-08-13,16,1351,204,25,76.11,https://wiki.openstreetmap.org/w/images/thumb/4/47/Building-levels.png/200px-Building-levels.png
+building:levels,fr,https://wiki.openstreetmap.org/wiki/FR:Key:building:levels,2024-08-01,15,1457,202,26,76.11,https://wiki.openstreetmap.org/w/images/thumb/4/47/Building-levels.png/200px-Building-levels.png
+amenity,en,https://wiki.openstreetmap.org/wiki/Key:amenity,2025-08-24,29,3066,915,504,160.78,https://wiki.openstreetmap.org/w/images/thumb/a/a5/Mapping-Features-Parking-Lot.png/200px-Mapping-Features-Parking-Lot.png
+amenity,fr,https://wiki.openstreetmap.org/wiki/FR:Key:amenity,2023-07-19,22,2146,800,487,160.78,https://wiki.openstreetmap.org/w/images/thumb/a/a5/Mapping-Features-Parking-Lot.png/200px-Mapping-Features-Parking-Lot.png
+barrier,en,https://wiki.openstreetmap.org/wiki/Key:barrier,2025-04-15,17,2137,443,173,207.98,https://upload.wikimedia.org/wikipedia/commons/thumb/4/4c/2014_Bystrzyca_K%C5%82odzka%2C_mury_obronne_05.jpg/200px-2014_Bystrzyca_K%C5%82odzka%2C_mury_obronne_05.jpg
+barrier,fr,https://wiki.openstreetmap.org/wiki/FR:Key:barrier,2022-08-16,15,542,103,18,207.98,https://upload.wikimedia.org/wikipedia/commons/thumb/4/4c/2014_Bystrzyca_K%C5%82odzka%2C_mury_obronne_05.jpg/200px-2014_Bystrzyca_K%C5%82odzka%2C_mury_obronne_05.jpg
+source:date,en,https://wiki.openstreetmap.org/wiki/Key:source:date,2023-04-01,11,395,75,10,22.47,https://wiki.openstreetmap.org/w/images/thumb/7/76/Osm_element_node.svg/30px-Osm_element_node.svg.png
+source:date,fr,https://wiki.openstreetmap.org/wiki/FR:Key:source:date,2023-07-21,10,419,75,11,22.47,https://wiki.openstreetmap.org/w/images/thumb/7/76/Osm_element_node.svg/30px-Osm_element_node.svg.png
+service,en,https://wiki.openstreetmap.org/wiki/Key:service,2025-03-16,22,1436,218,17,83.79,https://wiki.openstreetmap.org/w/images/thumb/7/76/Osm_element_node.svg/30px-Osm_element_node.svg.png
+service,fr,https://wiki.openstreetmap.org/wiki/FR:Key:service,2024-03-04,11,443,100,10,83.79,https://wiki.openstreetmap.org/w/images/thumb/7/76/Osm_element_node.svg/30px-Osm_element_node.svg.png
+addr:state,en,https://wiki.openstreetmap.org/wiki/Key:addr:state,2023-06-23,12,289,74,11,100,https://upload.wikimedia.org/wikipedia/commons/thumb/e/ef/WVaCent.jpg/200px-WVaCent.jpg