up wiki compare

wiki_compare/fetch_proposals.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import requests
+from bs4 import BeautifulSoup
+import json
+import logging
+import argparse
+import os
+from datetime import datetime, timedelta
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# URLs for OSM Wiki proposals
+VOTING_PROPOSALS_URL = "https://wiki.openstreetmap.org/wiki/Category:Proposals_with_%22Voting%22_status"
+RECENT_CHANGES_URL = "https://wiki.openstreetmap.org/w/index.php?title=Special:RecentChanges&namespace=102&limit=50"  # Namespace 102 is for Proposal pages
+
+# Output file
+OUTPUT_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'proposals.json')
+
+# Cache timeout (in hours)
+CACHE_TIMEOUT = 1
+
+def should_update_cache():
+    """
+    Check if the cache file exists and if it's older than the cache timeout
+    """
+    if not os.path.exists(OUTPUT_FILE):
+        logger.info("Cache file doesn't exist, creating it")
+        return True
+    
+    # Check file modification time
+    file_mtime = datetime.fromtimestamp(os.path.getmtime(OUTPUT_FILE))
+    now = datetime.now()
+    
+    # If file is older than cache timeout, update it
+    if now - file_mtime > timedelta(hours=CACHE_TIMEOUT):
+        logger.info(f"Cache is older than {CACHE_TIMEOUT} hour(s), updating")
+        return True
+    
+    logger.info(f"Cache is still fresh (less than {CACHE_TIMEOUT} hour(s) old)")
+    return False
+
+def fetch_voting_proposals():
+    """
+    Fetch proposals with "Voting" status from the OSM Wiki
+    """
+    logger.info(f"Fetching voting proposals from {VOTING_PROPOSALS_URL}")
+    try:
+        response = requests.get(VOTING_PROPOSALS_URL)
+        response.raise_for_status()
+        
+        soup = BeautifulSoup(response.text, 'html.parser')
+        proposals = []
+        
+        # Find all links in the mw-pages section
+        links = soup.select('#mw-pages a')
+        
+        for link in links:
+            # Skip category links and other non-proposal links
+            if 'Category:' in link.get('href', '') or 'Special:' in link.get('href', ''):
+                continue
+                
+            proposal_title = link.text.strip()
+            proposal_url = 'https://wiki.openstreetmap.org' + link.get('href', '')
+            
+            proposals.append({
+                'title': proposal_title,
+                'url': proposal_url,
+                'status': 'Voting',
+                'type': 'voting'
+            })
+        
+        logger.info(f"Found {len(proposals)} voting proposals")
+        return proposals
+    
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Error fetching voting proposals: {e}")
+        return []
+
+def fetch_recent_proposals():
+    """
+    Fetch recently modified proposals from the OSM Wiki
+    """
+    logger.info(f"Fetching recent changes from {RECENT_CHANGES_URL}")
+    try:
+        response = requests.get(RECENT_CHANGES_URL)
+        response.raise_for_status()
+        
+        soup = BeautifulSoup(response.text, 'html.parser')
+        proposals = []
+        
+        # Find all change list lines
+        change_lines = soup.select('.mw-changeslist .mw-changeslist-line')
+        
+        for line in change_lines:
+            # Get the page title
+            title_element = line.select_one('.mw-changeslist-title')
+            if not title_element:
+                continue
+                
+            page_title = title_element.text.strip()
+            page_url = title_element.get('href', '')
+            if not page_url.startswith('http'):
+                page_url = f"https://wiki.openstreetmap.org{page_url}"
+            
+            # Get the timestamp
+            timestamp_element = line.select_one('.mw-changeslist-date')
+            timestamp = timestamp_element.text.strip() if timestamp_element else ""
+            
+            # Get the user who made the change
+            user_element = line.select_one('.mw-userlink')
+            user = user_element.text.strip() if user_element else "Unknown"
+            
+            # Skip if it's not a proposal page
+            if not page_title.startswith('Proposal:'):
+                continue
+            
+            proposals.append({
+                'title': page_title,
+                'url': page_url,
+                'last_modified': timestamp,
+                'modified_by': user,
+                'type': 'recent'
+            })
+        
+        # Limit to the 10 most recent proposals
+        proposals = proposals[:10]
+        logger.info(f"Found {len(proposals)} recently modified proposals")
+        return proposals
+    
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Error fetching recent proposals: {e}")
+        return []
+
+def save_proposals(voting_proposals, recent_proposals):
+    """
+    Save the proposals to a JSON file
+    """
+    data = {
+        'last_updated': datetime.now().isoformat(),
+        'voting_proposals': voting_proposals,
+        'recent_proposals': recent_proposals
+    }
+    
+    with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+    
+    logger.info(f"Saved {len(voting_proposals)} voting proposals and {len(recent_proposals)} recent proposals to {OUTPUT_FILE}")
+    return OUTPUT_FILE
+
+def main():
+    parser = argparse.ArgumentParser(description='Fetch OSM Wiki proposals')
+    parser.add_argument('--force', action='store_true', help='Force update even if cache is fresh')
+    parser.add_argument('--dry-run', action='store_true', help='Print results without saving to file')
+    args = parser.parse_args()
+    
+    # Check if we should update the cache
+    if args.force or should_update_cache() or args.dry_run:
+        voting_proposals = fetch_voting_proposals()
+        recent_proposals = fetch_recent_proposals()
+        
+        if args.dry_run:
+            logger.info(f"Found {len(voting_proposals)} voting proposals:")
+            for proposal in voting_proposals:
+                logger.info(f"- {proposal['title']}")
+            
+            logger.info(f"Found {len(recent_proposals)} recent proposals:")
+            for proposal in recent_proposals:
+                logger.info(f"- {proposal['title']} (modified by {proposal['modified_by']} on {proposal['last_modified']})")
+        else:
+            output_file = save_proposals(voting_proposals, recent_proposals)
+            logger.info(f"Results saved to {output_file}")
+    else:
+        logger.info("Using cached proposals data")
+
+if __name__ == "__main__":
+    main()