up demo links and map controls
This commit is contained in:
Tykayn
tykayn
parent
f76bded4e4
commit
8613d218cd
5 changed files with 420 additions and 189 deletions
18
README.md
18
README.md
|
|
@ -1,4 +1,5 @@
|
|||
# OpenEventDatabase Backend
|
||||
|
||||
|
||||
OpenEventDatabase (OEDB) is a database for events with geographic information.
|
||||
It is a collaborative way to share things that have no space in OpenStreetMap.
|
||||
|
|
@ -156,10 +157,21 @@ créer une page de démo qui permet de modifier un évènement, faire un lien ve
|
|||
vérifier le fonctionnement des endpoints de recherche avec les queryparameters, les mettre dans la page de démo.
|
||||
|
||||
la page /demo/by-what a une erreur, Error: Expecting value: line 1 column 1 (char 0)
|
||||
récupérer les évènements depuis osmcal dans esm_cal.py
|
||||
dans les extracteurs, vérifier qu'il n'existe pas déjà des évènements avec les mês propriétés avant de les créer.
|
||||
récupérer les évènements depuis osmcal dans osm_cal.py ✓
|
||||
dans les extracteurs, vérifier qu'il n'existe pas déjà des évènements avec les mês propriétés avant de les créer. ✓
|
||||
|
||||
Error: Expecting value: line 1 column 1 (char 0)
|
||||
## Database Schema
|
||||
|
||||
The following diagram shows the database schema for the OpenEventDatabase:
|
||||
|
||||
|
||||
|
||||
The database consists of three main tables:
|
||||
- **events**: Stores event data including type, what, when, geo reference, and tags
|
||||
- **events_deleted**: Archive of deleted events
|
||||
- **geo**: Stores geometry data referenced by events
|
||||
|
||||
The events table has a foreign key relationship with the geo table through the events_geo field, which references the hash field in the geo table.
|
||||
|
||||
|
||||
-- il manque l'attribution openstreetmap sur les cartes maplibre. ✓
|
||||
|
|
|
|||
107
doc/database_schema.svg
Normal file
107
doc/database_schema.svg
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<svg width="800" height="600" xmlns="http://www.w3.org/2000/svg">
|
||||
<style>
|
||||
.table {
|
||||
fill: #f5f5f5;
|
||||
stroke: #333;
|
||||
stroke-width: 2;
|
||||
}
|
||||
.table-header {
|
||||
fill: #4285f4;
|
||||
stroke: #333;
|
||||
stroke-width: 2;
|
||||
}
|
||||
.table-text {
|
||||
font-family: Arial, sans-serif;
|
||||
font-size: 14px;
|
||||
fill: #333;
|
||||
}
|
||||
.header-text {
|
||||
font-family: Arial, sans-serif;
|
||||
font-size: 16px;
|
||||
font-weight: bold;
|
||||
fill: white;
|
||||
}
|
||||
.pk {
|
||||
font-weight: bold;
|
||||
}
|
||||
.fk {
|
||||
fill: #0066cc;
|
||||
}
|
||||
.arrow {
|
||||
stroke: #666;
|
||||
stroke-width: 2;
|
||||
fill: none;
|
||||
marker-end: url(#arrowhead);
|
||||
}
|
||||
.title {
|
||||
font-family: Arial, sans-serif;
|
||||
font-size: 24px;
|
||||
font-weight: bold;
|
||||
fill: #333;
|
||||
}
|
||||
</style>
|
||||
|
||||
<defs>
|
||||
<marker id="arrowhead" markerWidth="10" markerHeight="7" refX="9" refY="3.5" orient="auto">
|
||||
<polygon points="0 0, 10 3.5, 0 7" fill="#666" />
|
||||
</marker>
|
||||
</defs>
|
||||
|
||||
<text x="400" y="40" class="title" text-anchor="middle">OpenEventDatabase Schema</text>
|
||||
|
||||
<!-- events table -->
|
||||
<rect x="100" y="100" width="250" height="220" rx="5" ry="5" class="table" />
|
||||
<rect x="100" y="100" width="250" height="30" rx="5" ry="5" class="table-header" />
|
||||
<text x="225" y="120" class="header-text" text-anchor="middle">events</text>
|
||||
|
||||
<text x="110" y="150" class="table-text pk">events_id (uuid)</text>
|
||||
<text x="110" y="170" class="table-text">createdate (timestamp)</text>
|
||||
<text x="110" y="190" class="table-text">lastupdate (timestamp)</text>
|
||||
<text x="110" y="210" class="table-text">events_type (text)</text>
|
||||
<text x="110" y="230" class="table-text">events_what (text)</text>
|
||||
<text x="110" y="250" class="table-text">events_when (tstzrange)</text>
|
||||
<text x="110" y="270" class="table-text fk">events_geo (text) → geo.hash</text>
|
||||
<text x="110" y="290" class="table-text">events_tags (jsonb)</text>
|
||||
|
||||
<!-- events_deleted table -->
|
||||
<rect x="100" y="350" width="250" height="200" rx="5" ry="5" class="table" />
|
||||
<rect x="100" y="350" width="250" height="30" rx="5" ry="5" class="table-header" />
|
||||
<text x="225" y="370" class="header-text" text-anchor="middle">events_deleted</text>
|
||||
|
||||
<text x="110" y="400" class="table-text">events_id (uuid)</text>
|
||||
<text x="110" y="420" class="table-text">createdate (timestamp)</text>
|
||||
<text x="110" y="440" class="table-text">lastupdate (timestamp)</text>
|
||||
<text x="110" y="460" class="table-text">events_type (text)</text>
|
||||
<text x="110" y="480" class="table-text">events_what (text)</text>
|
||||
<text x="110" y="500" class="table-text">events_when (tstzrange)</text>
|
||||
<text x="110" y="520" class="table-text">events_geo (text)</text>
|
||||
<text x="110" y="540" class="table-text">events_tags (jsonb)</text>
|
||||
|
||||
<!-- geo table -->
|
||||
<rect x="450" y="150" width="250" height="140" rx="5" ry="5" class="table" />
|
||||
<rect x="450" y="150" width="250" height="30" rx="5" ry="5" class="table-header" />
|
||||
<text x="575" y="170" class="header-text" text-anchor="middle">geo</text>
|
||||
|
||||
<text x="460" y="200" class="table-text">geom (geometry)</text>
|
||||
<text x="460" y="220" class="table-text pk">hash (text)</text>
|
||||
<text x="460" y="240" class="table-text">geom_center (point)</text>
|
||||
<text x="460" y="260" class="table-text">idx (geometry)</text>
|
||||
|
||||
<!-- Relationship arrow -->
|
||||
<path d="M 350 270 L 400 270 L 400 220 L 450 220" class="arrow" />
|
||||
|
||||
<!-- Indexes -->
|
||||
<rect x="450" y="350" width="250" height="200" rx="5" ry="5" class="table" />
|
||||
<rect x="450" y="350" width="250" height="30" rx="5" ry="5" class="table-header" />
|
||||
<text x="575" y="370" class="header-text" text-anchor="middle">Indexes</text>
|
||||
|
||||
<text x="460" y="400" class="table-text">events_idx_antidup (unique)</text>
|
||||
<text x="460" y="420" class="table-text">events_idx_id (unique)</text>
|
||||
<text x="460" y="440" class="table-text">events_idx_lastupdate</text>
|
||||
<text x="460" y="460" class="table-text">events_idx_what (spgist)</text>
|
||||
<text x="460" y="480" class="table-text">events_idx_when (spgist)</text>
|
||||
<text x="460" y="500" class="table-text">geo_geom (gist)</text>
|
||||
<text x="460" y="520" class="table-text">geo_idx (gist)</text>
|
||||
<text x="460" y="540" class="table-text">events_idx_where_osm (spgist)</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 4.3 KiB |
391
extractors/osm_cal.py
Normal file → Executable file
391
extractors/osm_cal.py
Normal file → Executable file
|
|
@ -2,8 +2,8 @@
|
|||
"""
|
||||
OSM Calendar Extractor for the OpenEventDatabase.
|
||||
|
||||
This script fetches OpenStreetMap events from the osmcal.org RSS feed
|
||||
and adds them to the OpenEventDatabase.
|
||||
This script fetches events from the OpenStreetMap Calendar RSS feed
|
||||
and adds them to the OpenEventDatabase if they don't already exist.
|
||||
|
||||
RSS Feed URL: https://osmcal.org/events.rss
|
||||
"""
|
||||
|
|
@ -12,12 +12,10 @@ import json
|
|||
import requests
|
||||
import sys
|
||||
import os
|
||||
import feedparser
|
||||
from datetime import datetime, timedelta
|
||||
import pytz
|
||||
from dateutil import parser as date_parser
|
||||
import xml.etree.ElementTree as ET
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
import html
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
# Add the parent directory to the path so we can import from oedb
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
|
@ -25,143 +23,219 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')
|
|||
from oedb.utils.db import db_connect
|
||||
from oedb.utils.logging import logger
|
||||
|
||||
# RSS Feed URL for osmcal.org
|
||||
# RSS Feed URL for OSM Calendar
|
||||
RSS_URL = "https://osmcal.org/events.rss"
|
||||
|
||||
def fetch_osmcal_data():
|
||||
def fetch_osm_calendar_data():
|
||||
"""
|
||||
Fetch OpenStreetMap events from the osmcal.org RSS feed.
|
||||
Fetch events from the OSM Calendar RSS feed.
|
||||
|
||||
Returns:
|
||||
list: A list of event entries from the RSS feed.
|
||||
list: A list of event items from the RSS feed.
|
||||
"""
|
||||
logger.info("Fetching data from osmcal.org RSS feed")
|
||||
logger.info("Fetching data from OSM Calendar RSS feed")
|
||||
|
||||
try:
|
||||
# Parse the RSS feed
|
||||
feed = feedparser.parse(RSS_URL)
|
||||
response = requests.get(RSS_URL)
|
||||
response.raise_for_status() # Raise an exception for HTTP errors
|
||||
|
||||
if not feed.entries:
|
||||
logger.error("No entries found in RSS feed")
|
||||
# Parse the XML response
|
||||
root = ET.fromstring(response.content)
|
||||
|
||||
# Find all item elements (events)
|
||||
channel = root.find('channel')
|
||||
if channel is None:
|
||||
logger.error("No channel element found in RSS feed")
|
||||
return []
|
||||
|
||||
logger.success(f"Successfully fetched {len(feed.entries)} events from osmcal.org")
|
||||
return feed.entries
|
||||
items = channel.findall('item')
|
||||
|
||||
if not items:
|
||||
logger.error("No items found in RSS feed")
|
||||
return []
|
||||
|
||||
logger.success(f"Successfully fetched {len(items)} events from OSM Calendar RSS feed")
|
||||
return items
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"Error fetching data from OSM Calendar RSS feed: {e}")
|
||||
return []
|
||||
except ET.ParseError as e:
|
||||
logger.error(f"Error parsing XML response: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching data from osmcal.org: {e}")
|
||||
logger.error(f"Unexpected error fetching OSM Calendar data: {e}")
|
||||
return []
|
||||
|
||||
def extract_coordinates(location_str):
|
||||
def parse_event_dates(description):
|
||||
"""
|
||||
Extract coordinates from a location string.
|
||||
Parse event dates from the description.
|
||||
|
||||
Args:
|
||||
location_str (str): A string containing location information.
|
||||
description (str): The event description HTML.
|
||||
|
||||
Returns:
|
||||
tuple: A tuple containing (longitude, latitude) or None if not found.
|
||||
"""
|
||||
# Try to find coordinates in the format "lat,lon" or similar
|
||||
coord_pattern = r'(-?\d+\.\d+)[,\s]+(-?\d+\.\d+)'
|
||||
match = re.search(coord_pattern, location_str)
|
||||
|
||||
if match:
|
||||
lat = float(match.group(1))
|
||||
lon = float(match.group(2))
|
||||
return [lon, lat] # GeoJSON uses [longitude, latitude]
|
||||
|
||||
# Default coordinates (center of France) if none found
|
||||
return [2.2137, 46.2276]
|
||||
|
||||
def parse_date(date_str):
|
||||
"""
|
||||
Parse a date string into an ISO format string.
|
||||
|
||||
Args:
|
||||
date_str (str): A string containing date information.
|
||||
|
||||
Returns:
|
||||
str: An ISO format date string.
|
||||
tuple: A tuple containing (start_date, end_date) as ISO format strings.
|
||||
"""
|
||||
try:
|
||||
# Parse the date string
|
||||
dt = date_parser.parse(date_str)
|
||||
# Extract the date information from the description
|
||||
date_pattern = r'(\d+)(?:st|nd|rd|th)\s+(\w+)(?:\s+(\d+):(\d+)(?:\s+–\s+(\d+):(\d+))?)?(?:\s+\(([^)]+)\))?(?:\s+–\s+(\d+)(?:st|nd|rd|th)\s+(\w+))?'
|
||||
date_match = re.search(date_pattern, description)
|
||||
|
||||
# Ensure the datetime is timezone-aware
|
||||
if dt.tzinfo is None:
|
||||
dt = pytz.UTC.localize(dt)
|
||||
if not date_match:
|
||||
# Try alternative pattern for single day with time range
|
||||
date_pattern = r'(\d+)(?:st|nd|rd|th)\s+(\w+)\s+(\d+):(\d+)\s+–\s+(\d+):(\d+)'
|
||||
date_match = re.search(date_pattern, description)
|
||||
|
||||
return dt.isoformat()
|
||||
if date_match:
|
||||
# Extract date components
|
||||
day = int(date_match.group(1))
|
||||
month_name = date_match.group(2)
|
||||
|
||||
# Convert month name to month number
|
||||
month_map = {
|
||||
'January': 1, 'February': 2, 'March': 3, 'April': 4,
|
||||
'May': 5, 'June': 6, 'July': 7, 'August': 8,
|
||||
'September': 9, 'October': 10, 'November': 11, 'December': 12
|
||||
}
|
||||
|
||||
# Try to match the month name (case insensitive)
|
||||
month = None
|
||||
for name, num in month_map.items():
|
||||
if month_name.lower() == name.lower():
|
||||
month = num
|
||||
break
|
||||
|
||||
if month is None:
|
||||
# If month name not found, use current month
|
||||
month = datetime.now().month
|
||||
logger.warning(f"Could not parse month name: {month_name}, using current month")
|
||||
|
||||
# Get current year (assuming events are current or future)
|
||||
current_year = datetime.now().year
|
||||
|
||||
# Create start date
|
||||
try:
|
||||
start_date = datetime(current_year, month, day)
|
||||
except ValueError:
|
||||
# Handle invalid dates (e.g., February 30)
|
||||
logger.warning(f"Invalid date: {day} {month_name} {current_year}, using current date")
|
||||
start_date = datetime.now()
|
||||
|
||||
# Check if there's an end date
|
||||
if len(date_match.groups()) >= 8 and date_match.group(8):
|
||||
end_day = int(date_match.group(8))
|
||||
end_month_name = date_match.group(9)
|
||||
|
||||
# Convert end month name to month number
|
||||
end_month = None
|
||||
for name, num in month_map.items():
|
||||
if end_month_name.lower() == name.lower():
|
||||
end_month = num
|
||||
break
|
||||
|
||||
if end_month is None:
|
||||
# If end month name not found, use start month
|
||||
end_month = month
|
||||
logger.warning(f"Could not parse end month name: {end_month_name}, using start month")
|
||||
|
||||
try:
|
||||
end_date = datetime(current_year, end_month, end_day)
|
||||
# Add a day to include the full end day
|
||||
end_date = end_date + timedelta(days=1)
|
||||
except ValueError:
|
||||
# Handle invalid dates
|
||||
logger.warning(f"Invalid end date: {end_day} {end_month_name} {current_year}, using start date + 1 day")
|
||||
end_date = start_date + timedelta(days=1)
|
||||
else:
|
||||
# If no end date, use start date + 1 day as default
|
||||
end_date = start_date + timedelta(days=1)
|
||||
|
||||
# Format dates as ISO strings
|
||||
start_iso = start_date.isoformat()
|
||||
end_iso = end_date.isoformat()
|
||||
|
||||
return (start_iso, end_iso)
|
||||
else:
|
||||
# If no date pattern found, use current date as fallback
|
||||
now = datetime.now()
|
||||
start_iso = now.isoformat()
|
||||
end_iso = (now + timedelta(days=1)).isoformat()
|
||||
logger.warning(f"Could not parse date from description, using current date: {start_iso} to {end_iso}")
|
||||
return (start_iso, end_iso)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing date '{date_str}': {e}")
|
||||
# Return current date as fallback
|
||||
return datetime.now(pytz.UTC).isoformat()
|
||||
logger.error(f"Error parsing event dates: {e}")
|
||||
# Return default dates (current date)
|
||||
now = datetime.now()
|
||||
return (now.isoformat(), (now + timedelta(days=1)).isoformat())
|
||||
|
||||
def create_event(entry):
|
||||
def extract_location(description):
|
||||
"""
|
||||
Create an event object from an RSS feed entry.
|
||||
Extract location information from the event description.
|
||||
|
||||
Args:
|
||||
entry: An entry from the osmcal.org RSS feed.
|
||||
description (str): The event description HTML.
|
||||
|
||||
Returns:
|
||||
tuple: A tuple containing (location_name, coordinates).
|
||||
"""
|
||||
try:
|
||||
# Default coordinates (center of the world)
|
||||
coordinates = [0, 0]
|
||||
location_name = "Unknown Location"
|
||||
|
||||
# Try to find location in the description
|
||||
location_pattern = r'<p>([^<]+)</p>'
|
||||
location_matches = re.findall(location_pattern, description)
|
||||
|
||||
if location_matches and len(location_matches) > 1:
|
||||
# The second paragraph often contains the location
|
||||
location_candidate = location_matches[1].strip()
|
||||
if location_candidate and "," in location_candidate and not location_candidate.startswith('<'):
|
||||
location_name = location_candidate
|
||||
|
||||
# For now, we don't have exact coordinates, so we'll use a placeholder
|
||||
# In a real implementation, you might want to geocode the location
|
||||
coordinates = [0, 0]
|
||||
|
||||
return (location_name, coordinates)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error extracting location: {e}")
|
||||
return ("Unknown Location", [0, 0])
|
||||
|
||||
def create_event(item):
|
||||
"""
|
||||
Create an event object from an RSS item.
|
||||
|
||||
Args:
|
||||
item: An item element from the RSS feed.
|
||||
|
||||
Returns:
|
||||
dict: A GeoJSON Feature representing the event.
|
||||
"""
|
||||
try:
|
||||
# Extract data from the entry
|
||||
title = entry.title
|
||||
link = entry.link
|
||||
description = entry.description if hasattr(entry, 'description') else ""
|
||||
# Extract data from the item
|
||||
title = item.find('title').text
|
||||
link = item.find('link').text
|
||||
description = item.find('description').text
|
||||
guid = item.find('guid').text
|
||||
|
||||
# Extract dates
|
||||
start_date = None
|
||||
end_date = None
|
||||
# Clean up the description (remove HTML tags for text extraction)
|
||||
clean_description = re.sub(r'<[^>]+>', ' ', description)
|
||||
clean_description = html.unescape(clean_description)
|
||||
clean_description = re.sub(r'\s+', ' ', clean_description).strip()
|
||||
|
||||
if hasattr(entry, 'published'):
|
||||
start_date = parse_date(entry.published)
|
||||
# Parse dates from the description
|
||||
start_date, end_date = parse_event_dates(description)
|
||||
|
||||
# If there's no published date, use the current date
|
||||
if not start_date:
|
||||
start_date = datetime.now(pytz.UTC).isoformat()
|
||||
|
||||
# Set end date to 1 day after start date if not specified
|
||||
if not end_date:
|
||||
dt = date_parser.parse(start_date)
|
||||
end_date = (dt + timedelta(days=1)).isoformat()
|
||||
|
||||
# Extract location and coordinates
|
||||
location = ""
|
||||
coordinates = [2.2137, 46.2276] # Default: center of France
|
||||
|
||||
if hasattr(entry, 'where') and entry.where:
|
||||
location = entry.where
|
||||
coordinates = extract_coordinates(location)
|
||||
elif description:
|
||||
# Try to extract location from description
|
||||
location_match = re.search(r'Location:?\s*([^\n]+)', description, re.IGNORECASE)
|
||||
if location_match:
|
||||
location = location_match.group(1).strip()
|
||||
coordinates = extract_coordinates(location)
|
||||
# Extract location information
|
||||
location_name, coordinates = extract_location(description)
|
||||
|
||||
# Create a descriptive label
|
||||
label = title
|
||||
|
||||
# Determine the event type
|
||||
what = "community.osm.meetup"
|
||||
|
||||
# Check for specific event types in the title or description
|
||||
lower_title = title.lower()
|
||||
lower_desc = description.lower()
|
||||
|
||||
if any(term in lower_title or term in lower_desc for term in ["conference", "summit"]):
|
||||
what = "community.osm.conference"
|
||||
elif any(term in lower_title or term in lower_desc for term in ["workshop", "training"]):
|
||||
what = "community.osm.workshop"
|
||||
elif any(term in lower_title or term in lower_desc for term in ["mapathon", "mapping party"]):
|
||||
what = "community.osm.mapathon"
|
||||
|
||||
# Create the event object
|
||||
event = {
|
||||
"type": "Feature",
|
||||
|
|
@ -171,24 +245,74 @@ def create_event(entry):
|
|||
},
|
||||
"properties": {
|
||||
"type": "scheduled",
|
||||
"what": what,
|
||||
"what:series": "OpenStreetMap Events",
|
||||
"where": location,
|
||||
"what": "community.osm.event",
|
||||
"what:series": "OpenStreetMap Calendar",
|
||||
"where": location_name,
|
||||
"label": label,
|
||||
"description": description,
|
||||
"description": clean_description,
|
||||
"start": start_date,
|
||||
"stop": end_date,
|
||||
"url": link,
|
||||
"source": "osmcal.org"
|
||||
"external_id": guid,
|
||||
"source": "OSM Calendar"
|
||||
}
|
||||
}
|
||||
|
||||
return event
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating event from entry: {e}")
|
||||
logger.error(f"Error creating event from item: {e}")
|
||||
return None
|
||||
|
||||
def event_exists(db, properties):
|
||||
"""
|
||||
Check if an event with the same properties already exists in the database.
|
||||
|
||||
Args:
|
||||
db: Database connection.
|
||||
properties: Event properties.
|
||||
|
||||
Returns:
|
||||
bool: True if the event exists, False otherwise.
|
||||
"""
|
||||
try:
|
||||
cur = db.cursor()
|
||||
|
||||
# Check if an event with the same external_id exists
|
||||
if 'external_id' in properties:
|
||||
cur.execute("""
|
||||
SELECT events_id FROM events
|
||||
WHERE events_tags->>'external_id' = %s;
|
||||
""", (properties['external_id'],))
|
||||
|
||||
result = cur.fetchone()
|
||||
if result:
|
||||
logger.info(f"Event with external_id {properties['external_id']} already exists")
|
||||
return True
|
||||
|
||||
# Check if an event with the same label, start, and stop exists
|
||||
cur.execute("""
|
||||
SELECT events_id FROM events
|
||||
WHERE events_tags->>'label' = %s
|
||||
AND events_tags->>'start' = %s
|
||||
AND events_tags->>'stop' = %s;
|
||||
""", (
|
||||
properties.get('label', ''),
|
||||
properties.get('start', ''),
|
||||
properties.get('stop', '')
|
||||
))
|
||||
|
||||
result = cur.fetchone()
|
||||
if result:
|
||||
logger.info(f"Event with label '{properties.get('label')}' and same dates already exists")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking if event exists: {e}")
|
||||
return False
|
||||
|
||||
def submit_event(event):
|
||||
"""
|
||||
Submit an event to the OpenEventDatabase.
|
||||
|
|
@ -202,10 +326,17 @@ def submit_event(event):
|
|||
try:
|
||||
# Connect to the database
|
||||
db = db_connect()
|
||||
cur = db.cursor()
|
||||
|
||||
# Extract event properties
|
||||
properties = event['properties']
|
||||
|
||||
# Check if the event already exists
|
||||
if event_exists(db, properties):
|
||||
logger.info(f"Skipping event '{properties.get('label')}' as it already exists")
|
||||
db.close()
|
||||
return False
|
||||
|
||||
cur = db.cursor()
|
||||
geometry = json.dumps(event['geometry'])
|
||||
|
||||
# Insert the geometry into the geo table
|
||||
|
|
@ -239,27 +370,6 @@ def submit_event(event):
|
|||
# Determine the bounds for the time range
|
||||
bounds = '[]' if properties['start'] == properties['stop'] else '[)'
|
||||
|
||||
# Check if an event with the same properties already exists
|
||||
cur.execute("""
|
||||
SELECT events_id FROM events
|
||||
WHERE events_what = %s
|
||||
AND events_when = tstzrange(%s, %s, %s)
|
||||
AND events_geo = %s;
|
||||
""", (
|
||||
properties['what'],
|
||||
properties['start'],
|
||||
properties['stop'],
|
||||
bounds,
|
||||
geo_hash
|
||||
))
|
||||
|
||||
existing_id = cur.fetchone()
|
||||
|
||||
if existing_id:
|
||||
logger.info(f"Event already exists with ID: {existing_id[0]}")
|
||||
db.close()
|
||||
return False
|
||||
|
||||
# Insert the event into the database
|
||||
cur.execute("""
|
||||
INSERT INTO events (events_type, events_what, events_when, events_tags, events_geo)
|
||||
|
|
@ -294,23 +404,22 @@ def submit_event(event):
|
|||
|
||||
def main():
|
||||
"""
|
||||
Main function to fetch OSM Calendar data and add events to the database.
|
||||
Main function to fetch OSM Calendar events and add them to the database.
|
||||
"""
|
||||
logger.info("Starting OSM Calendar extractor")
|
||||
|
||||
# Fetch data from osmcal.org
|
||||
entries = fetch_osmcal_data()
|
||||
# Fetch events from the OSM Calendar RSS feed
|
||||
items = fetch_osm_calendar_data()
|
||||
|
||||
if not entries:
|
||||
logger.warning("No entries found, exiting")
|
||||
if not items:
|
||||
logger.warning("No events found, exiting")
|
||||
return
|
||||
|
||||
# Process each entry
|
||||
# Process each item
|
||||
success_count = 0
|
||||
|
||||
for entry in entries:
|
||||
# Create an event from the entry
|
||||
event = create_event(entry)
|
||||
for item in items:
|
||||
# Create an event from the item
|
||||
event = create_event(item)
|
||||
|
||||
if not event:
|
||||
continue
|
||||
|
|
@ -319,7 +428,7 @@ def main():
|
|||
if submit_event(event):
|
||||
success_count += 1
|
||||
|
||||
logger.success(f"Successfully added {success_count} events to the database")
|
||||
logger.success(f"Successfully added {success_count} out of {len(items)} events to the database")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,6 +1,9 @@
|
|||
falcon
|
||||
psycopg2-binary
|
||||
geojson
|
||||
gunicorn
|
||||
uwsgi
|
||||
requests
|
||||
beautifulsoup4==4.13.5
|
||||
config==0.5.1
|
||||
falcon==4.1.0
|
||||
iso8601==2.1.0
|
||||
psycopg2_binary==2.9.10
|
||||
pyproj==3.7.2
|
||||
pytz==2025.2
|
||||
Requests==2.32.5
|
||||
waitress==3.0.2
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ Ce document explique comment installer et activer le service systemd pour faire
|
|||
```bash
|
||||
sudo cp oedb-uwsgi.service /etc/systemd/system/
|
||||
sudo chmod 644 /etc/systemd/system/oedb-uwsgi.service
|
||||
sudo chown -R www-data:www-data /home/poule/encrypted/stockage-syncable/www/development/html/oedb-backend
|
||||
sudo chown -R www-data:www-data /home/poule/encrypted/oedb-backend
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable oedb-uwsgi.service
|
||||
sudo systemctl start oedb-uwsgi.service
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue