up wiki compare

wiki_compare/fetch_osm_fr_groups.py

@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+fetch_osm_fr_groups.py
+
+This script scrapes the OpenStreetMap wiki page for France/OSM-FR to extract
+information about local working groups. It specifically targets links in the
+#Pages_des_groupes_locaux section.
+
+Usage:
+    python fetch_osm_fr_groups.py [--dry-run] [--force]
+
+Options:
+    --dry-run    Run the script without saving the results to a file
+    --force      Force update even if the cache is still fresh (less than 1 hour old)
+
+Output:
+    - osm_fr_groups.json: JSON file with information about OSM-FR local groups
+    - Log messages about the scraping process and results
+"""
+
+import json
+import argparse
+import logging
+import os
+from datetime import datetime, timedelta
+import requests
+from bs4 import BeautifulSoup
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S'
+)
+logger = logging.getLogger(__name__)
+
+# Constants
+OUTPUT_FILE = "osm_fr_groups.json"
+BASE_URL = "https://wiki.openstreetmap.org/wiki/France/OSM-FR"
+WIKI_BASE_URL = "https://wiki.openstreetmap.org"
+CACHE_DURATION = timedelta(hours=1)  # Cache duration of 1 hour
+
+def is_cache_fresh():
+    """
+    Check if the cache file exists and is less than CACHE_DURATION old
+    
+    Returns:
+        bool: True if cache is fresh, False otherwise
+    """
+    if not os.path.exists(OUTPUT_FILE):
+        return False
+    
+    try:
+        with open(OUTPUT_FILE, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+            last_updated = datetime.fromisoformat(data.get('last_updated', '2000-01-01T00:00:00'))
+            now = datetime.now()
+            return (now - last_updated) < CACHE_DURATION
+    except (IOError, json.JSONDecodeError, ValueError) as e:
+        logger.error(f"Error checking cache freshness: {e}")
+        return False
+
+def get_page_content(url):
+    """
+    Get the HTML content of a page
+    
+    Args:
+        url (str): URL to fetch
+        
+    Returns:
+        str: HTML content of the page or None if request failed
+    """
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        return response.text
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Error fetching {url}: {e}")
+        return None
+
+def extract_working_groups(html_content):
+    """
+    Extract working groups from the wiki page HTML
+    
+    Args:
+        html_content (str): HTML content of the wiki page
+        
+    Returns:
+        list: List of working group dictionaries
+    """
+    if not html_content:
+        return []
+    
+    soup = BeautifulSoup(html_content, 'html.parser')
+    working_groups = []
+    
+    # Find the working groups section
+    working_groups_section = None
+    for heading in soup.find_all(['h2', 'h3']):
+        if heading.get_text().strip() == 'Groupes de travail' or 'Groupes_de_travail' in heading.get_text():
+            working_groups_section = heading
+            break
+    
+    if not working_groups_section:
+        logger.warning("Could not find working groups section")
+        # Return an empty list but with a default category
+        return []
+    
+    # Get the content following the heading until the next heading
+    current = working_groups_section.next_sibling
+    while current and not current.name in ['h2', 'h3']:
+        if current.name == 'ul':
+            # Process list items
+            for li in current.find_all('li', recursive=False):
+                link = li.find('a')
+                if link:
+                    name = link.get_text().strip()
+                    url = WIKI_BASE_URL + link.get('href') if link.get('href').startswith('/') else link.get('href')
+                    
+                    # Extract description (text after the link)
+                    description = ""
+                    next_node = link.next_sibling
+                    while next_node:
+                        if isinstance(next_node, str):
+                            description += next_node.strip()
+                        next_node = next_node.next_sibling if hasattr(next_node, 'next_sibling') else None
+                    
+                    description = description.strip(' :-,')
+                    
+                    working_groups.append({
+                        "name": name,
+                        "url": url,
+                        "description": description,
+                        "category": "Général",
+                        "type": "working_group"
+                    })
+        current = current.next_sibling
+    
+    logger.info(f"Found {len(working_groups)} working groups")
+    return working_groups
+
+def extract_local_groups(html_content):
+    """
+    Extract local groups from the wiki page HTML
+    
+    Args:
+        html_content (str): HTML content of the wiki page
+        
+    Returns:
+        list: List of local group dictionaries
+    """
+    if not html_content:
+        return []
+    
+    soup = BeautifulSoup(html_content, 'html.parser')
+    local_groups = []
+    
+    # Find the local groups section
+    local_groups_section = None
+    for heading in soup.find_all(['h2', 'h3']):
+        if heading.get_text().strip() == 'Groupes locaux' or 'Pages des groupes locaux' in heading.get_text():
+            local_groups_section = heading
+            break
+    
+    if not local_groups_section:
+        logger.warning("Could not find local groups section")
+        return []
+    
+    # Get the content following the heading until the next heading
+    current = local_groups_section.next_sibling
+    while current and not current.name in ['h2', 'h3']:
+        if current.name == 'ul':
+            # Process list items
+            for li in current.find_all('li', recursive=False):
+                link = li.find('a')
+                if link:
+                    name = link.get_text().strip()
+                    url = WIKI_BASE_URL + link.get('href') if link.get('href').startswith('/') else link.get('href')
+                    
+                    # Extract description (text after the link)
+                    description = ""
+                    next_node = link.next_sibling
+                    while next_node:
+                        if isinstance(next_node, str):
+                            description += next_node.strip()
+                        next_node = next_node.next_sibling if hasattr(next_node, 'next_sibling') else None
+                    
+                    description = description.strip(' :-,')
+                    
+                    local_groups.append({
+                        "name": name,
+                        "url": url,
+                        "description": description,
+                        "type": "local_group"
+                    })
+        current = current.next_sibling
+    
+    logger.info(f"Found {len(local_groups)} local groups")
+    return local_groups
+
+def extract_umap_url(html_content):
+    """
+    Extract the uMap URL for OSM-FR local groups
+    
+    Args:
+        html_content (str): HTML content of the wiki page
+        
+    Returns:
+        str: uMap URL or None if not found
+    """
+    if not html_content:
+        return None
+    
+    soup = BeautifulSoup(html_content, 'html.parser')
+    
+    # Look for links to umap.openstreetmap.fr
+    for link in soup.find_all('a'):
+        href = link.get('href', '')
+        if 'umap.openstreetmap.fr' in href and 'groupes-locaux' in href:
+            return href
+    
+    return None
+
+def save_results(local_groups, working_groups, umap_url, dry_run=False):
+    """
+    Save the results to a JSON file
+    
+    Args:
+        local_groups (list): List of local group dictionaries
+        working_groups (list): List of working group dictionaries
+        umap_url (str): URL to the uMap for local groups
+        dry_run (bool): If True, don't actually save to file
+        
+    Returns:
+        bool: True if saving was successful or dry run, False otherwise
+    """
+    if dry_run:
+        logger.info("DRY RUN: Would have saved results to file")
+        logger.info(f"Local groups: {len(local_groups)}")
+        for group in local_groups:
+            logger.info(f"  - {group['name']}: {group['url']}")
+        logger.info(f"Working groups: {len(working_groups)}")
+        for group in working_groups:
+            logger.info(f"  - {group['name']}: {group['url']}")
+        if umap_url:
+            logger.info(f"uMap URL: {umap_url}")
+        return True
+    
+    # Prepare the data structure
+    data = {
+        "last_updated": datetime.now().isoformat(),
+        "local_groups": local_groups,
+        "working_groups": working_groups,
+        "umap_url": umap_url
+    }
+    
+    try:
+        with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
+            json.dump(data, f, indent=2, ensure_ascii=False)
+        logger.info(f"Successfully saved {len(local_groups)} local groups and {len(working_groups)} working groups to {OUTPUT_FILE}")
+        return True
+    except IOError as e:
+        logger.error(f"Error saving results to {OUTPUT_FILE}: {e}")
+        return False
+
+def main():
+    """Main function to execute the script"""
+    parser = argparse.ArgumentParser(description="Scrape OSM-FR local groups from the wiki")
+    parser.add_argument("--dry-run", action="store_true", help="Run without saving results to file")
+    parser.add_argument("--force", action="store_true", help="Force update even if cache is fresh")
+    args = parser.parse_args()
+    
+    logger.info("Starting fetch_osm_fr_groups.py")
+    
+    # Check if cache is fresh
+    if is_cache_fresh() and not args.force:
+        logger.info(f"Cache is still fresh (less than {CACHE_DURATION.total_seconds()/3600} hours old)")
+        logger.info(f"Use --force to update anyway")
+        return
+    
+    # Get the wiki page content
+    html_content = get_page_content(BASE_URL)
+    
+    if not html_content:
+        logger.error("Failed to get wiki page content")
+        return
+    
+    # Extract local groups
+    local_groups = extract_local_groups(html_content)
+    
+    if not local_groups:
+        logger.warning("No local groups found")
+    
+    # Extract working groups
+    working_groups = extract_working_groups(html_content)
+    
+    if not working_groups:
+        logger.warning("No working groups found")
+        # Initialize with an empty list to avoid errors in the controller
+        working_groups = []
+    
+    # Extract uMap URL
+    umap_url = extract_umap_url(html_content)
+    
+    # Save results
+    success = save_results(local_groups, working_groups, umap_url, args.dry_run)
+    
+    if success:
+        logger.info("Script completed successfully")
+    else:
+        logger.error("Script completed with errors")
+
+if __name__ == "__main__":
+    main()