osm-commerces/wiki_compare/fetch_proposals.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import requests
from bs4 import BeautifulSoup
import json
import logging
import argparse
import os
from datetime import datetime, timedelta

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

# URLs for OSM Wiki proposals
VOTING_PROPOSALS_URL = "https://wiki.openstreetmap.org/wiki/Category:Proposals_with_%22Voting%22_status"
RECENT_CHANGES_URL = "https://wiki.openstreetmap.org/w/index.php?title=Special:RecentChanges&namespace=102&limit=50"  # Namespace 102 is for Proposal pages

# Output file
OUTPUT_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'proposals.json')

# Cache timeout (in hours)
CACHE_TIMEOUT = 1

def should_update_cache():
    """
    Check if the cache file exists and if it's older than the cache timeout
    """
    if not os.path.exists(OUTPUT_FILE):
        logger.info("Cache file doesn't exist, creating it")
        return True
    
    # Check file modification time
    file_mtime = datetime.fromtimestamp(os.path.getmtime(OUTPUT_FILE))
    now = datetime.now()
    
    # If file is older than cache timeout, update it
    if now - file_mtime > timedelta(hours=CACHE_TIMEOUT):
        logger.info(f"Cache is older than {CACHE_TIMEOUT} hour(s), updating")
        return True
    
    logger.info(f"Cache is still fresh (less than {CACHE_TIMEOUT} hour(s) old)")
    return False

def fetch_voting_proposals():
    """
    Fetch proposals with "Voting" status from the OSM Wiki
    """
    logger.info(f"Fetching voting proposals from {VOTING_PROPOSALS_URL}")
    try:
        response = requests.get(VOTING_PROPOSALS_URL)
        response.raise_for_status()
        
        soup = BeautifulSoup(response.text, 'html.parser')
        proposals = []
        
        # Find all links in the mw-pages section
        links = soup.select('#mw-pages a')
        
        for link in links:
            # Skip category links and other non-proposal links
            if 'Category:' in link.get('href', '') or 'Special:' in link.get('href', ''):
                continue
                
            proposal_title = link.text.strip()
            proposal_url = 'https://wiki.openstreetmap.org' + link.get('href', '')
            
            proposals.append({
                'title': proposal_title,
                'url': proposal_url,
                'status': 'Voting',
                'type': 'voting'
            })
        
        logger.info(f"Found {len(proposals)} voting proposals")
        return proposals
    
    except requests.exceptions.RequestException as e:
        logger.error(f"Error fetching voting proposals: {e}")
        return []

def fetch_recent_proposals():
    """
    Fetch recently modified proposals from the OSM Wiki
    """
    logger.info(f"Fetching recent changes from {RECENT_CHANGES_URL}")
    try:
        response = requests.get(RECENT_CHANGES_URL)
        response.raise_for_status()
        
        soup = BeautifulSoup(response.text, 'html.parser')
        proposals = []
        
        # Find all change list lines
        change_lines = soup.select('.mw-changeslist .mw-changeslist-line')
        
        for line in change_lines:
            # Get the page title
            title_element = line.select_one('.mw-changeslist-title')
            if not title_element:
                continue
                
            page_title = title_element.text.strip()
            page_url = title_element.get('href', '')
            if not page_url.startswith('http'):
                page_url = f"https://wiki.openstreetmap.org{page_url}"
            
            # Get the timestamp
            timestamp_element = line.select_one('.mw-changeslist-date')
            timestamp = timestamp_element.text.strip() if timestamp_element else ""
            
            # Get the user who made the change
            user_element = line.select_one('.mw-userlink')
            user = user_element.text.strip() if user_element else "Unknown"
            
            # Skip if it's not a proposal page
            if not page_title.startswith('Proposal:'):
                continue
            
            proposals.append({
                'title': page_title,
                'url': page_url,
                'last_modified': timestamp,
                'modified_by': user,
                'type': 'recent'
            })
        
        # Limit to the 10 most recent proposals
        proposals = proposals[:10]
        logger.info(f"Found {len(proposals)} recently modified proposals")
        return proposals
    
    except requests.exceptions.RequestException as e:
        logger.error(f"Error fetching recent proposals: {e}")
        return []

def save_proposals(voting_proposals, recent_proposals):
    """
    Save the proposals to a JSON file
    """
    data = {
        'last_updated': datetime.now().isoformat(),
        'voting_proposals': voting_proposals,
        'recent_proposals': recent_proposals
    }
    
    with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
        json.dump(data, f, ensure_ascii=False, indent=2)
    
    logger.info(f"Saved {len(voting_proposals)} voting proposals and {len(recent_proposals)} recent proposals to {OUTPUT_FILE}")
    return OUTPUT_FILE

def main():
    parser = argparse.ArgumentParser(description='Fetch OSM Wiki proposals')
    parser.add_argument('--force', action='store_true', help='Force update even if cache is fresh')
    parser.add_argument('--dry-run', action='store_true', help='Print results without saving to file')
    args = parser.parse_args()
    
    # Check if we should update the cache
    if args.force or should_update_cache() or args.dry_run:
        voting_proposals = fetch_voting_proposals()
        recent_proposals = fetch_recent_proposals()
        
        if args.dry_run:
            logger.info(f"Found {len(voting_proposals)} voting proposals:")
            for proposal in voting_proposals:
                logger.info(f"- {proposal['title']}")
            
            logger.info(f"Found {len(recent_proposals)} recent proposals:")
            for proposal in recent_proposals:
                logger.info(f"- {proposal['title']} (modified by {proposal['modified_by']} on {proposal['last_modified']})")
        else:
            output_file = save_proposals(voting_proposals, recent_proposals)
            logger.info(f"Results saved to {output_file}")
    else:
        logger.info("Using cached proposals data")

if __name__ == "__main__":
    main()