up demo links and map controls

README.md

@@ -1,4 +1,5 @@
 # OpenEventDatabase Backend
+![oedb.png](oedb.png)
 
 OpenEventDatabase (OEDB) is a database for events with geographic information.
 It is a collaborative way to share things that have no space in OpenStreetMap.
@@ -156,10 +157,21 @@ créer une page de démo qui permet de modifier un évènement, faire un lien ve
 vérifier le fonctionnement des endpoints de recherche avec les queryparameters, les mettre dans la page de démo.
 
 la page /demo/by-what a une erreur, Error: Expecting value: line 1 column 1 (char 0)
-récupérer les évènements depuis osmcal dans esm_cal.py
+récupérer les évènements depuis osmcal dans osm_cal.py ✓
-dans les extracteurs, vérifier qu'il n'existe pas déjà des évènements avec les mês propriétés avant de les créer.
+dans les extracteurs, vérifier qu'il n'existe pas déjà des évènements avec les mês propriétés avant de les créer. ✓
 
-Error: Expecting value: line 1 column 1 (char 0)
+## Database Schema
+
+The following diagram shows the database schema for the OpenEventDatabase:
+
+![Database Schema](doc/database_schema.svg)
+
+The database consists of three main tables:
+- **events**: Stores event data including type, what, when, geo reference, and tags
+- **events_deleted**: Archive of deleted events
+- **geo**: Stores geometry data referenced by events
+
+The events table has a foreign key relationship with the geo table through the events_geo field, which references the hash field in the geo table.
 
 
 -- il manque l'attribution openstreetmap sur les cartes maplibre. ✓

doc/database_schema.svg

@@ -0,0 +1,107 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg width="800" height="600" xmlns="http://www.w3.org/2000/svg">
+  <style>
+    .table {
+      fill: #f5f5f5;
+      stroke: #333;
+      stroke-width: 2;
+    }
+    .table-header {
+      fill: #4285f4;
+      stroke: #333;
+      stroke-width: 2;
+    }
+    .table-text {
+      font-family: Arial, sans-serif;
+      font-size: 14px;
+      fill: #333;
+    }
+    .header-text {
+      font-family: Arial, sans-serif;
+      font-size: 16px;
+      font-weight: bold;
+      fill: white;
+    }
+    .pk {
+      font-weight: bold;
+    }
+    .fk {
+      fill: #0066cc;
+    }
+    .arrow {
+      stroke: #666;
+      stroke-width: 2;
+      fill: none;
+      marker-end: url(#arrowhead);
+    }
+    .title {
+      font-family: Arial, sans-serif;
+      font-size: 24px;
+      font-weight: bold;
+      fill: #333;
+    }
+  </style>
+  
+  <defs>
+    <marker id="arrowhead" markerWidth="10" markerHeight="7" refX="9" refY="3.5" orient="auto">
+      <polygon points="0 0, 10 3.5, 0 7" fill="#666" />
+    </marker>
+  </defs>
+  
+  <text x="400" y="40" class="title" text-anchor="middle">OpenEventDatabase Schema</text>
+  
+  <!-- events table -->
+  <rect x="100" y="100" width="250" height="220" rx="5" ry="5" class="table" />
+  <rect x="100" y="100" width="250" height="30" rx="5" ry="5" class="table-header" />
+  <text x="225" y="120" class="header-text" text-anchor="middle">events</text>
+  
+  <text x="110" y="150" class="table-text pk">events_id (uuid)</text>
+  <text x="110" y="170" class="table-text">createdate (timestamp)</text>
+  <text x="110" y="190" class="table-text">lastupdate (timestamp)</text>
+  <text x="110" y="210" class="table-text">events_type (text)</text>
+  <text x="110" y="230" class="table-text">events_what (text)</text>
+  <text x="110" y="250" class="table-text">events_when (tstzrange)</text>
+  <text x="110" y="270" class="table-text fk">events_geo (text) → geo.hash</text>
+  <text x="110" y="290" class="table-text">events_tags (jsonb)</text>
+  
+  <!-- events_deleted table -->
+  <rect x="100" y="350" width="250" height="200" rx="5" ry="5" class="table" />
+  <rect x="100" y="350" width="250" height="30" rx="5" ry="5" class="table-header" />
+  <text x="225" y="370" class="header-text" text-anchor="middle">events_deleted</text>
+  
+  <text x="110" y="400" class="table-text">events_id (uuid)</text>
+  <text x="110" y="420" class="table-text">createdate (timestamp)</text>
+  <text x="110" y="440" class="table-text">lastupdate (timestamp)</text>
+  <text x="110" y="460" class="table-text">events_type (text)</text>
+  <text x="110" y="480" class="table-text">events_what (text)</text>
+  <text x="110" y="500" class="table-text">events_when (tstzrange)</text>
+  <text x="110" y="520" class="table-text">events_geo (text)</text>
+  <text x="110" y="540" class="table-text">events_tags (jsonb)</text>
+  
+  <!-- geo table -->
+  <rect x="450" y="150" width="250" height="140" rx="5" ry="5" class="table" />
+  <rect x="450" y="150" width="250" height="30" rx="5" ry="5" class="table-header" />
+  <text x="575" y="170" class="header-text" text-anchor="middle">geo</text>
+  
+  <text x="460" y="200" class="table-text">geom (geometry)</text>
+  <text x="460" y="220" class="table-text pk">hash (text)</text>
+  <text x="460" y="240" class="table-text">geom_center (point)</text>
+  <text x="460" y="260" class="table-text">idx (geometry)</text>
+  
+  <!-- Relationship arrow -->
+  <path d="M 350 270 L 400 270 L 400 220 L 450 220" class="arrow" />
+  
+  <!-- Indexes -->
+  <rect x="450" y="350" width="250" height="200" rx="5" ry="5" class="table" />
+  <rect x="450" y="350" width="250" height="30" rx="5" ry="5" class="table-header" />
+  <text x="575" y="370" class="header-text" text-anchor="middle">Indexes</text>
+  
+  <text x="460" y="400" class="table-text">events_idx_antidup (unique)</text>
+  <text x="460" y="420" class="table-text">events_idx_id (unique)</text>
+  <text x="460" y="440" class="table-text">events_idx_lastupdate</text>
+  <text x="460" y="460" class="table-text">events_idx_what (spgist)</text>
+  <text x="460" y="480" class="table-text">events_idx_when (spgist)</text>
+  <text x="460" y="500" class="table-text">geo_geom (gist)</text>
+  <text x="460" y="520" class="table-text">geo_idx (gist)</text>
+  <text x="460" y="540" class="table-text">events_idx_where_osm (spgist)</text>
+</svg>

extractors/osm_cal.py

@@ -2,8 +2,8 @@
 """
 OSM Calendar Extractor for the OpenEventDatabase.
 
-This script fetches OpenStreetMap events from the osmcal.org RSS feed
+This script fetches events from the OpenStreetMap Calendar RSS feed
-and adds them to the OpenEventDatabase.
+and adds them to the OpenEventDatabase if they don't already exist.
 
 RSS Feed URL: https://osmcal.org/events.rss
 """
@@ -12,12 +12,10 @@ import json
 import requests
 import sys
 import os
-import feedparser
+import xml.etree.ElementTree as ET
-from datetime import datetime, timedelta
-import pytz
-from dateutil import parser as date_parser
 import re
-from urllib.parse import urlparse
+import html
+from datetime import datetime, timedelta
 
 # Add the parent directory to the path so we can import from oedb
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
@@ -25,143 +23,219 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')
 from oedb.utils.db import db_connect
 from oedb.utils.logging import logger
 
-# RSS Feed URL for osmcal.org
+# RSS Feed URL for OSM Calendar
 RSS_URL = "https://osmcal.org/events.rss"
 
-def fetch_osmcal_data():
+def fetch_osm_calendar_data():
     """
-    Fetch OpenStreetMap events from the osmcal.org RSS feed.
+    Fetch events from the OSM Calendar RSS feed.
 
     Returns:
-        list: A list of event entries from the RSS feed.
+        list: A list of event items from the RSS feed.
     """
-    logger.info("Fetching data from osmcal.org RSS feed")
+    logger.info("Fetching data from OSM Calendar RSS feed")
 
     try:
-        # Parse the RSS feed
+        response = requests.get(RSS_URL)
-        feed = feedparser.parse(RSS_URL)
+        response.raise_for_status()  # Raise an exception for HTTP errors
 
-        if not feed.entries:
+        # Parse the XML response
-            logger.error("No entries found in RSS feed")
+        root = ET.fromstring(response.content)
+
+        # Find all item elements (events)
+        channel = root.find('channel')
+        if channel is None:
+            logger.error("No channel element found in RSS feed")
             return []
 
-        logger.success(f"Successfully fetched {len(feed.entries)} events from osmcal.org")
+        items = channel.findall('item')
-        return feed.entries
 
+        if not items:
+            logger.error("No items found in RSS feed")
+            return []
+
+        logger.success(f"Successfully fetched {len(items)} events from OSM Calendar RSS feed")
+        return items
+
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Error fetching data from OSM Calendar RSS feed: {e}")
+        return []
+    except ET.ParseError as e:
+        logger.error(f"Error parsing XML response: {e}")
+        return []
     except Exception as e:
-        logger.error(f"Error fetching data from osmcal.org: {e}")
+        logger.error(f"Unexpected error fetching OSM Calendar data: {e}")
         return []
 
-def extract_coordinates(location_str):
+def parse_event_dates(description):
     """
-    Extract coordinates from a location string.
+    Parse event dates from the description.
 
     Args:
-        location_str (str): A string containing location information.
+        description (str): The event description HTML.
 
     Returns:
-        tuple: A tuple containing (longitude, latitude) or None if not found.
+        tuple: A tuple containing (start_date, end_date) as ISO format strings.
-    """
-    # Try to find coordinates in the format "lat,lon" or similar
-    coord_pattern = r'(-?\d+\.\d+)[,\s]+(-?\d+\.\d+)'
-    match = re.search(coord_pattern, location_str)
-    
-    if match:
-        lat = float(match.group(1))
-        lon = float(match.group(2))
-        return [lon, lat]  # GeoJSON uses [longitude, latitude]
-    
-    # Default coordinates (center of France) if none found
-    return [2.2137, 46.2276]
-
-def parse_date(date_str):
-    """
-    Parse a date string into an ISO format string.
-    
-    Args:
-        date_str (str): A string containing date information.
-        
-    Returns:
-        str: An ISO format date string.
     """
     try:
-        # Parse the date string
+        # Extract the date information from the description
-        dt = date_parser.parse(date_str)
+        date_pattern = r'(\d+)(?:st|nd|rd|th)\s+(\w+)(?:\s+(\d+):(\d+)(?:\s+–\s+(\d+):(\d+))?)?(?:\s+\(([^)]+)\))?(?:\s+–\s+(\d+)(?:st|nd|rd|th)\s+(\w+))?'
+        date_match = re.search(date_pattern, description)
 
-        # Ensure the datetime is timezone-aware
+        if not date_match:
-        if dt.tzinfo is None:
+            # Try alternative pattern for single day with time range
-            dt = pytz.UTC.localize(dt)
+            date_pattern = r'(\d+)(?:st|nd|rd|th)\s+(\w+)\s+(\d+):(\d+)\s+–\s+(\d+):(\d+)'
+            date_match = re.search(date_pattern, description)
 
-        return dt.isoformat()
+        if date_match:
+            # Extract date components
+            day = int(date_match.group(1))
+            month_name = date_match.group(2)
+
+            # Convert month name to month number
+            month_map = {
+                'January': 1, 'February': 2, 'March': 3, 'April': 4,
+                'May': 5, 'June': 6, 'July': 7, 'August': 8,
+                'September': 9, 'October': 10, 'November': 11, 'December': 12
+            }
+
+            # Try to match the month name (case insensitive)
+            month = None
+            for name, num in month_map.items():
+                if month_name.lower() == name.lower():
+                    month = num
+                    break
+
+            if month is None:
+                # If month name not found, use current month
+                month = datetime.now().month
+                logger.warning(f"Could not parse month name: {month_name}, using current month")
+
+            # Get current year (assuming events are current or future)
+            current_year = datetime.now().year
+
+            # Create start date
+            try:
+                start_date = datetime(current_year, month, day)
+            except ValueError:
+                # Handle invalid dates (e.g., February 30)
+                logger.warning(f"Invalid date: {day} {month_name} {current_year}, using current date")
+                start_date = datetime.now()
+
+            # Check if there's an end date
+            if len(date_match.groups()) >= 8 and date_match.group(8):
+                end_day = int(date_match.group(8))
+                end_month_name = date_match.group(9)
+
+                # Convert end month name to month number
+                end_month = None
+                for name, num in month_map.items():
+                    if end_month_name.lower() == name.lower():
+                        end_month = num
+                        break
+
+                if end_month is None:
+                    # If end month name not found, use start month
+                    end_month = month
+                    logger.warning(f"Could not parse end month name: {end_month_name}, using start month")
+
+                try:
+                    end_date = datetime(current_year, end_month, end_day)
+                    # Add a day to include the full end day
+                    end_date = end_date + timedelta(days=1)
+                except ValueError:
+                    # Handle invalid dates
+                    logger.warning(f"Invalid end date: {end_day} {end_month_name} {current_year}, using start date + 1 day")
+                    end_date = start_date + timedelta(days=1)
+            else:
+                # If no end date, use start date + 1 day as default
+                end_date = start_date + timedelta(days=1)
+
+            # Format dates as ISO strings
+            start_iso = start_date.isoformat()
+            end_iso = end_date.isoformat()
+
+            return (start_iso, end_iso)
+        else:
+            # If no date pattern found, use current date as fallback
+            now = datetime.now()
+            start_iso = now.isoformat()
+            end_iso = (now + timedelta(days=1)).isoformat()
+            logger.warning(f"Could not parse date from description, using current date: {start_iso} to {end_iso}")
+            return (start_iso, end_iso)
 
     except Exception as e:
-        logger.error(f"Error parsing date '{date_str}': {e}")
+        logger.error(f"Error parsing event dates: {e}")
-        # Return current date as fallback
+        # Return default dates (current date)
-        return datetime.now(pytz.UTC).isoformat()
+        now = datetime.now()
+        return (now.isoformat(), (now + timedelta(days=1)).isoformat())
 
-def create_event(entry):
+def extract_location(description):
     """
-    Create an event object from an RSS feed entry.
+    Extract location information from the event description.
 
     Args:
-        entry: An entry from the osmcal.org RSS feed.
+        description (str): The event description HTML.
+
+    Returns:
+        tuple: A tuple containing (location_name, coordinates).
+    """
+    try:
+        # Default coordinates (center of the world)
+        coordinates = [0, 0]
+        location_name = "Unknown Location"
+
+        # Try to find location in the description
+        location_pattern = r'<p>([^<]+)</p>'
+        location_matches = re.findall(location_pattern, description)
+
+        if location_matches and len(location_matches) > 1:
+            # The second paragraph often contains the location
+            location_candidate = location_matches[1].strip()
+            if location_candidate and "," in location_candidate and not location_candidate.startswith('<'):
+                location_name = location_candidate
+
+                # For now, we don't have exact coordinates, so we'll use a placeholder
+                # In a real implementation, you might want to geocode the location
+                coordinates = [0, 0]
+
+        return (location_name, coordinates)
+
+    except Exception as e:
+        logger.error(f"Error extracting location: {e}")
+        return ("Unknown Location", [0, 0])
+
+def create_event(item):
+    """
+    Create an event object from an RSS item.
+
+    Args:
+        item: An item element from the RSS feed.
 
     Returns:
         dict: A GeoJSON Feature representing the event.
     """
     try:
-        # Extract data from the entry
+        # Extract data from the item
-        title = entry.title
+        title = item.find('title').text
-        link = entry.link
+        link = item.find('link').text
-        description = entry.description if hasattr(entry, 'description') else ""
+        description = item.find('description').text
+        guid = item.find('guid').text
 
-        # Extract dates
+        # Clean up the description (remove HTML tags for text extraction)
-        start_date = None
+        clean_description = re.sub(r'<[^>]+>', ' ', description)
-        end_date = None
+        clean_description = html.unescape(clean_description)
+        clean_description = re.sub(r'\s+', ' ', clean_description).strip()
 
-        if hasattr(entry, 'published'):
+        # Parse dates from the description
-            start_date = parse_date(entry.published)
+        start_date, end_date = parse_event_dates(description)
 
-        # If there's no published date, use the current date
+        # Extract location information
-        if not start_date:
+        location_name, coordinates = extract_location(description)
-            start_date = datetime.now(pytz.UTC).isoformat()
-        
-        # Set end date to 1 day after start date if not specified
-        if not end_date:
-            dt = date_parser.parse(start_date)
-            end_date = (dt + timedelta(days=1)).isoformat()
-        
-        # Extract location and coordinates
-        location = ""
-        coordinates = [2.2137, 46.2276]  # Default: center of France
-        
-        if hasattr(entry, 'where') and entry.where:
-            location = entry.where
-            coordinates = extract_coordinates(location)
-        elif description:
-            # Try to extract location from description
-            location_match = re.search(r'Location:?\s*([^\n]+)', description, re.IGNORECASE)
-            if location_match:
-                location = location_match.group(1).strip()
-                coordinates = extract_coordinates(location)
 
         # Create a descriptive label
         label = title
 
-        # Determine the event type
-        what = "community.osm.meetup"
-        
-        # Check for specific event types in the title or description
-        lower_title = title.lower()
-        lower_desc = description.lower()
-        
-        if any(term in lower_title or term in lower_desc for term in ["conference", "summit"]):
-            what = "community.osm.conference"
-        elif any(term in lower_title or term in lower_desc for term in ["workshop", "training"]):
-            what = "community.osm.workshop"
-        elif any(term in lower_title or term in lower_desc for term in ["mapathon", "mapping party"]):
-            what = "community.osm.mapathon"
-        
         # Create the event object
         event = {
             "type": "Feature",
@@ -171,24 +245,74 @@ def create_event(entry):
             },
             "properties": {
                 "type": "scheduled",
-                "what": what,
+                "what": "community.osm.event",
-                "what:series": "OpenStreetMap Events",
+                "what:series": "OpenStreetMap Calendar",
-                "where": location,
+                "where": location_name,
                 "label": label,
-                "description": description,
+                "description": clean_description,
                 "start": start_date,
                 "stop": end_date,
                 "url": link,
-                "source": "osmcal.org"
+                "external_id": guid,
+                "source": "OSM Calendar"
             }
         }
 
         return event
 
     except Exception as e:
-        logger.error(f"Error creating event from entry: {e}")
+        logger.error(f"Error creating event from item: {e}")
         return None
 
+def event_exists(db, properties):
+    """
+    Check if an event with the same properties already exists in the database.
+
+    Args:
+        db: Database connection.
+        properties: Event properties.
+
+    Returns:
+        bool: True if the event exists, False otherwise.
+    """
+    try:
+        cur = db.cursor()
+
+        # Check if an event with the same external_id exists
+        if 'external_id' in properties:
+            cur.execute("""
+                SELECT events_id FROM events
+                WHERE events_tags->>'external_id' = %s;
+            """, (properties['external_id'],))
+
+            result = cur.fetchone()
+            if result:
+                logger.info(f"Event with external_id {properties['external_id']} already exists")
+                return True
+
+        # Check if an event with the same label, start, and stop exists
+        cur.execute("""
+            SELECT events_id FROM events
+            WHERE events_tags->>'label' = %s
+            AND events_tags->>'start' = %s
+            AND events_tags->>'stop' = %s;
+        """, (
+            properties.get('label', ''),
+            properties.get('start', ''),
+            properties.get('stop', '')
+        ))
+
+        result = cur.fetchone()
+        if result:
+            logger.info(f"Event with label '{properties.get('label')}' and same dates already exists")
+            return True
+
+        return False
+
+    except Exception as e:
+        logger.error(f"Error checking if event exists: {e}")
+        return False
+
 def submit_event(event):
     """
     Submit an event to the OpenEventDatabase.
@@ -202,10 +326,17 @@ def submit_event(event):
     try:
         # Connect to the database
         db = db_connect()
-        cur = db.cursor()
 
         # Extract event properties
         properties = event['properties']
+
+        # Check if the event already exists
+        if event_exists(db, properties):
+            logger.info(f"Skipping event '{properties.get('label')}' as it already exists")
+            db.close()
+            return False
+
+        cur = db.cursor()
         geometry = json.dumps(event['geometry'])
 
         # Insert the geometry into the geo table
@@ -239,27 +370,6 @@ def submit_event(event):
         # Determine the bounds for the time range
         bounds = '[]' if properties['start'] == properties['stop'] else '[)'
 
-        # Check if an event with the same properties already exists
-        cur.execute("""
-            SELECT events_id FROM events
-            WHERE events_what = %s
-            AND events_when = tstzrange(%s, %s, %s)
-            AND events_geo = %s;
-        """, (
-            properties['what'],
-            properties['start'],
-            properties['stop'],
-            bounds,
-            geo_hash
-        ))
-        
-        existing_id = cur.fetchone()
-        
-        if existing_id:
-            logger.info(f"Event already exists with ID: {existing_id[0]}")
-            db.close()
-            return False
-        
         # Insert the event into the database
         cur.execute("""
             INSERT INTO events (events_type, events_what, events_when, events_tags, events_geo)
@@ -294,23 +404,22 @@ def submit_event(event):
 
 def main():
     """
-    Main function to fetch OSM Calendar data and add events to the database.
+    Main function to fetch OSM Calendar events and add them to the database.
     """
     logger.info("Starting OSM Calendar extractor")
 
-    # Fetch data from osmcal.org
+    # Fetch events from the OSM Calendar RSS feed
-    entries = fetch_osmcal_data()
+    items = fetch_osm_calendar_data()
 
-    if not entries:
+    if not items:
-        logger.warning("No entries found, exiting")
+        logger.warning("No events found, exiting")
         return
 
-    # Process each entry
+    # Process each item
     success_count = 0
-    
+    for item in items:
-    for entry in entries:
+        # Create an event from the item
-        # Create an event from the entry
+        event = create_event(item)
-        event = create_event(entry)
 
         if not event:
             continue
@@ -319,7 +428,7 @@ def main():
         if submit_event(event):
             success_count += 1
 
-    logger.success(f"Successfully added {success_count} events to the database")
+    logger.success(f"Successfully added {success_count} out of {len(items)} events to the database")
 
 if __name__ == "__main__":
     main()

requirements.txt

@@ -1,6 +1,9 @@
-falcon
+beautifulsoup4==4.13.5
-psycopg2-binary
+config==0.5.1
-geojson
+falcon==4.1.0
-gunicorn
+iso8601==2.1.0
-uwsgi
+psycopg2_binary==2.9.10
-requests
+pyproj==3.7.2
+pytz==2025.2
+Requests==2.32.5
+waitress==3.0.2

server_config/SYSTEMD_SERVICE_INSTALLATION.md

@@ -18,7 +18,7 @@ Ce document explique comment installer et activer le service systemd pour faire
 ```bash
 sudo cp oedb-uwsgi.service /etc/systemd/system/
 sudo chmod 644 /etc/systemd/system/oedb-uwsgi.service
-sudo chown -R www-data:www-data /home/poule/encrypted/stockage-syncable/www/development/html/oedb-backend
+sudo chown -R www-data:www-data /home/poule/encrypted/oedb-backend
 sudo systemctl daemon-reload
 sudo systemctl enable oedb-uwsgi.service
 sudo systemctl start oedb-uwsgi.service