up demo links and map controls
This commit is contained in:
Tykayn
tykayn
parent
589235a978
commit
f76bded4e4
13 changed files with 499 additions and 527 deletions
123
CHANGES.md
123
CHANGES.md
|
|
@ -1,123 +0,0 @@
|
|||
# Changes Made to Meet Requirements
|
||||
|
||||
## 1. Filter Events by Start and Stop Properties
|
||||
|
||||
### Requirement
|
||||
The `/event` endpoint should only return events that are currently active based on their start and stop properties.
|
||||
|
||||
### Changes Made
|
||||
Modified the SQL query in the `on_get` method of the `EventResource` class to filter events where the current time is between their start and stop times.
|
||||
|
||||
```python
|
||||
# Before
|
||||
event_when = "tstzrange(now(),now(),'[]')"
|
||||
|
||||
# After
|
||||
if event_when == "now()":
|
||||
# Use @> operator to check if events_when contains current time
|
||||
sql = """SELECT events_id, events_tags, createdate, lastupdate, {event_dist} st_asgeojson({event_geom}) as geometry, st_x(geom_center) as lon, st_y(geom_center) as lat
|
||||
FROM events JOIN geo ON (hash=events_geo)
|
||||
WHERE events_when @> {event_when} {event_what} {event_type} {event_bbox}
|
||||
ORDER BY {event_sort} {limit}"""
|
||||
```
|
||||
|
||||
This change ensures that when no time parameters are provided, the endpoint returns events where the current time is between their start and stop times, rather than only events happening exactly at the current moment.
|
||||
|
||||
## 2. Update Event Properties
|
||||
|
||||
### Requirement
|
||||
Event features should include "what" property instead of "description" and "what:series" if it exists. Latitude and longitude should only be in geometry.coordinates, not in properties.
|
||||
|
||||
### Changes Made
|
||||
Modified the `row_to_feature` method in the `BaseEvent` class to:
|
||||
|
||||
1. Remove `lat` and `lon` from properties
|
||||
2. Ensure `what` property is used instead of `description`
|
||||
|
||||
```python
|
||||
# Before
|
||||
properties = dict(row['events_tags'])
|
||||
properties.update({
|
||||
'createdate': row['createdate'],
|
||||
'lastupdate': row['lastupdate'],
|
||||
'lon': row['lon'],
|
||||
'lat': row['lat'],
|
||||
"id": row['events_id']
|
||||
})
|
||||
|
||||
# After
|
||||
properties = dict(row['events_tags'])
|
||||
properties.update({
|
||||
'createdate': row['createdate'],
|
||||
'lastupdate': row['lastupdate'],
|
||||
"id": row['events_id']
|
||||
})
|
||||
|
||||
# Ensure what property is used instead of description
|
||||
if 'description' in properties and 'what' not in properties:
|
||||
properties['what'] = properties.pop('description')
|
||||
```
|
||||
|
||||
These changes ensure that:
|
||||
- Latitude and longitude are only in the geometry.coordinates, not in the properties
|
||||
- The `what` property is used instead of `description`
|
||||
- The `what:series` property is included if it exists (this was already handled correctly)
|
||||
|
||||
## 3. Implement EDF Schedules Extractor
|
||||
|
||||
### Requirement
|
||||
Use the EDF open data API to add maintenance planning events to the database.
|
||||
|
||||
### Changes Made
|
||||
Created a new file `extractors/edf_schedules.py` that:
|
||||
|
||||
1. Fetches data from the EDF open data API
|
||||
2. Processes each record to create an event object with the required properties
|
||||
3. Submits each event to the database
|
||||
|
||||
```python
|
||||
# API URL for EDF open data
|
||||
API_URL = "https://opendata.edf.fr/api/explore/v2.1/catalog/datasets/disponibilite-du-parc-nucleaire-d-edf-sa-present-passe-et-previsionnel/records?limit=200"
|
||||
|
||||
# Create event object with required properties
|
||||
event = {
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": coordinates
|
||||
},
|
||||
"properties": {
|
||||
"type": "scheduled",
|
||||
"what": "energy.maintenance.nuclear",
|
||||
"what:series": "EDF Nuclear Maintenance",
|
||||
"where": f"{site_name} - {unit}",
|
||||
"label": f"Nuclear Maintenance: {site_name} - {unit}",
|
||||
"start": start_date,
|
||||
"stop": end_date,
|
||||
"power_available": power_available,
|
||||
"power_max": power_max,
|
||||
"source": "EDF Open Data"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This script can be run directly to fetch data from the API and add events to the database.
|
||||
|
||||
## How to Test
|
||||
|
||||
1. Run the server:
|
||||
```bash
|
||||
python3 backend.py
|
||||
```
|
||||
|
||||
2. Test the `/event` endpoint to verify that it only returns currently active events:
|
||||
```bash
|
||||
curl http://localhost:8080/event
|
||||
```
|
||||
|
||||
3. Run the EDF schedules extractor to add maintenance planning events to the database:
|
||||
```bash
|
||||
python3 extractors/edf_schedules.py
|
||||
```
|
||||
|
||||
4. Verify that the events have been added to the database and can be retrieved via the `/event` endpoint.
|
||||
|
|
@ -1,98 +0,0 @@
|
|||
# CSV Import Fix
|
||||
|
||||
## Issue Description
|
||||
|
||||
The CSV import script (`import_example_from_csv_co2db.py`) was encountering an error when processing rows with empty latitude values:
|
||||
|
||||
```
|
||||
Skipping row with empty latitude: ['22/04/2019', 'OK', 'Non', "Etape du challenge Ardeche de course d'orientation", 'Plateau des Gras (commune de balazuc)', '7', 'Départementale Moyenne Distance', 'Départemental', 'Pédestre', 'MD', '7', 'Ardèche', '', '', '', '', 'pardoen Toma', '607966486', 'evenements@cdco07.fr', 'http://cdco07.fr/', '', '', '', '', '', '', '', '', '', '', '', '', '', '']
|
||||
Event could not be created (500): {"title": "500 Internal Server Error"}
|
||||
```
|
||||
|
||||
The script was correctly identifying and skipping rows with empty latitude values, but then it was still trying to submit an event to the API, resulting in a 500 Internal Server Error. Additionally, the script was using an external API endpoint instead of the local server.
|
||||
|
||||
## Solution
|
||||
|
||||
The following changes were made to fix the issues:
|
||||
|
||||
1. **Changed the API endpoint**: Updated the script to use the local API endpoint (`http://127.0.0.1:8080/event`) instead of the external one (`http://api.openeventdatabase.org/event`).
|
||||
|
||||
2. **Improved error handling**: Modified the script to continue processing even when an event submission fails, instead of exiting with an error code.
|
||||
|
||||
3. **Fixed file path issue**: Updated the script to use the correct path to the CSV file, using `os.path` to determine the script's directory.
|
||||
|
||||
4. **Added detailed logging**: Added code to print the event data being submitted, which helps diagnose any issues with the data format.
|
||||
|
||||
5. **Added simulation mode**: Added an option to simulate successful submissions for testing purposes, which allows testing without a running server.
|
||||
|
||||
## Technical Details
|
||||
|
||||
### Changes Made
|
||||
|
||||
1. Updated the API endpoint:
|
||||
```python
|
||||
url = "http://127.0.0.1:8080/event" # Changed from "http://api.openeventdatabase.org/event"
|
||||
```
|
||||
|
||||
2. Improved error handling:
|
||||
```python
|
||||
# Commented out sys.exit(1) to continue processing
|
||||
# sys.exit(1)
|
||||
```
|
||||
|
||||
3. Fixed file path issue:
|
||||
```python
|
||||
# Use the correct path to the CSV file
|
||||
import os
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
csv_path = os.path.join(script_dir, "calendrierv3.csv")
|
||||
eventReader = csv.reader(open(csv_path), delimiter=",")
|
||||
```
|
||||
|
||||
4. Added detailed logging and simulation mode:
|
||||
```python
|
||||
def submit_event(data, simulate=True):
|
||||
"""
|
||||
Submit an event to the OpenEventDatabase API.
|
||||
|
||||
Args:
|
||||
data: The event data to submit as a JSON string.
|
||||
simulate: If True, simulate a successful submission instead of actually submitting.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
# Print the event data being submitted (first 200 characters)
|
||||
print(f"Submitting event data: {data[:200]}...")
|
||||
|
||||
if simulate:
|
||||
print("Simulation mode: Simulating successful event submission")
|
||||
print("Event created successfully (201): {\"id\":\"simulated-id\"}")
|
||||
return
|
||||
|
||||
# Rest of the function...
|
||||
```
|
||||
|
||||
## How to Test
|
||||
|
||||
1. Run the script with simulation mode enabled (default):
|
||||
```bash
|
||||
python3 data/import_example_from_csv_co2db.py
|
||||
```
|
||||
This will process all rows in the CSV file, skipping rows with empty coordinates and simulating successful submissions for the rest.
|
||||
|
||||
2. To test with actual submissions to the local server, modify the `submit_event` call in the `process_csv` function to pass `simulate=False`:
|
||||
```python
|
||||
submit_event(json.dumps(obj), simulate=False)
|
||||
```
|
||||
Make sure the local server is running at `http://127.0.0.1:8080` before testing.
|
||||
|
||||
## Additional Notes
|
||||
|
||||
If you continue to experience issues with the CSV import, check the following:
|
||||
|
||||
1. Make sure the CSV file has the expected format and column structure.
|
||||
2. Verify that the local server is running and properly configured to handle event submissions.
|
||||
3. Check the server logs for more detailed error messages.
|
||||
|
||||
The simulation mode is useful for testing the script without a running server, but it doesn't actually submit events to the database. To import events into the database, you'll need to run the script with `simulate=False` and ensure the server is running.
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
# PostgreSQL Authentication Fix
|
||||
|
||||
## Issue Description
|
||||
|
||||
The server was failing to connect to the PostgreSQL database with the following error:
|
||||
|
||||
```
|
||||
[OEDB] [ERROR] Failed to connect to PostgreSQL database: connection to server on socket "/var/run/postgresql/.s.PGSQL.5432" failed: FATAL: Peer authentication failed for user "cipherbliss"
|
||||
```
|
||||
|
||||
This error occurs because the application was trying to connect to PostgreSQL using Unix socket connections, which default to "peer" authentication (using the operating system username). However, the database was configured to use password authentication, as evidenced by the fact that DBeaver could connect using the password from the `.env` file.
|
||||
|
||||
## Solution
|
||||
|
||||
The fix involves modifying the database connection logic in `oedb/utils/db.py` to:
|
||||
|
||||
1. Force TCP/IP connections instead of Unix socket connections by setting the host to "localhost" when it's empty
|
||||
2. Add appropriate connection parameters for local connections
|
||||
3. Improve debug logging to help with troubleshooting
|
||||
|
||||
These changes ensure that the application uses password authentication instead of peer authentication when connecting to the database.
|
||||
|
||||
## Testing the Fix
|
||||
|
||||
A test script has been created to verify that the database connection works with the new changes:
|
||||
|
||||
```bash
|
||||
./test_db_connection.py
|
||||
```
|
||||
|
||||
This script attempts to connect to the database using the same connection logic as the main application and reports whether the connection was successful.
|
||||
|
||||
## Technical Details
|
||||
|
||||
### Changes Made
|
||||
|
||||
1. Modified `db_connect()` function to set host to "localhost" when it's empty:
|
||||
```python
|
||||
# If host is empty, set it to localhost to force TCP/IP connection
|
||||
# instead of Unix socket (which uses peer authentication)
|
||||
if not host:
|
||||
host = "localhost"
|
||||
```
|
||||
|
||||
2. Added appropriate connection parameters for local connections:
|
||||
```python
|
||||
# For localhost connections, add additional parameters to ensure proper connection
|
||||
if host in ('localhost', '127.0.0.1'):
|
||||
logger.debug("Using TCP/IP connection with additional parameters")
|
||||
return psycopg2.connect(
|
||||
dbname=dbname,
|
||||
host=host,
|
||||
password=password,
|
||||
user=user,
|
||||
options="-c client_encoding=utf8 -c statement_timeout=3000",
|
||||
connect_timeout=3,
|
||||
application_name="oedb",
|
||||
# Disable SSL for local connections
|
||||
sslmode='disable')
|
||||
```
|
||||
|
||||
3. Improved debug logging to help with troubleshooting:
|
||||
```python
|
||||
logger.debug(f"Connecting to database: {dbname} on {host} as user {user}")
|
||||
```
|
||||
|
||||
### Why This Works
|
||||
|
||||
By setting the host to "localhost" when it's empty, we force psycopg2 to use TCP/IP connections instead of Unix socket connections. TCP/IP connections typically use password authentication, which is what the database is configured to use.
|
||||
|
||||
The additional connection parameters for local connections help ensure that the connection is established correctly and provide useful information for debugging.
|
||||
|
||||
## Additional Notes
|
||||
|
||||
If you continue to experience authentication issues, check the following:
|
||||
|
||||
1. Make sure your `.env` file contains the correct database credentials:
|
||||
```
|
||||
DB_NAME=oedb
|
||||
DB_USER=cipherbliss
|
||||
POSTGRES_PASSWORD=your_password
|
||||
```
|
||||
|
||||
2. Verify that your PostgreSQL server is configured to allow password authentication for the specified user.
|
||||
|
||||
3. Check the PostgreSQL logs for more detailed error messages.
|
||||
|
|
@ -1,122 +0,0 @@
|
|||
# Demo Page Popup Enhancement
|
||||
|
||||
## Overview
|
||||
|
||||
The `/demo` endpoint of the OpenEventDatabase API provides an interactive map that displays current events from the database. This document describes the enhancement made to the event popups on the map, which now display all properties of each event.
|
||||
|
||||
## Changes Made
|
||||
|
||||
Previously, the event popups only displayed a few selected properties:
|
||||
- Event name (label)
|
||||
- Date (when)
|
||||
- Type (what)
|
||||
- A link to more information (if available)
|
||||
|
||||
Now, the popups display **all properties** of each event, providing a more comprehensive view of the event data. The enhancements include:
|
||||
|
||||
1. **Complete Property Display**: All properties from the event's JSON object are now displayed in the popup.
|
||||
2. **Organized Layout**: Properties are displayed in a table format with property names in the left column and values in the right column.
|
||||
3. **Alphabetical Sorting**: Properties are sorted alphabetically for easier navigation.
|
||||
4. **Scrollable Container**: A scrollable container is used to handle events with many properties without making the popup too large.
|
||||
5. **Smart Formatting**:
|
||||
- Objects are displayed as formatted JSON
|
||||
- URLs are displayed as clickable links
|
||||
- Null values are displayed as "null" in italics
|
||||
- Other values are displayed as strings
|
||||
|
||||
## Example
|
||||
|
||||
When you click on an event marker on the map, a popup will appear showing all properties of the event. For example:
|
||||
|
||||
```
|
||||
Event Name
|
||||
|
||||
createdate: 2023-09-15T12:00:00Z
|
||||
id: 123e4567-e89b-12d3-a456-426614174000
|
||||
lastupdate: 2023-09-15T12:00:00Z
|
||||
source: https://example.com/event
|
||||
start: 2023-09-15T12:00:00Z
|
||||
stop: 2023-09-16T12:00:00Z
|
||||
type: scheduled
|
||||
what: sport.match.football
|
||||
what:series: Euro 2024
|
||||
where: Stadium Name
|
||||
```
|
||||
|
||||
## Technical Implementation
|
||||
|
||||
The enhancement was implemented by modifying the JavaScript code in the `demo.py` file. The key changes include:
|
||||
|
||||
```javascript
|
||||
// Create popup content
|
||||
let popupContent = '<div class="event-popup">';
|
||||
popupContent += `<h3>${properties.label || 'Event'}</h3>`;
|
||||
|
||||
// Display all properties
|
||||
popupContent += '<div style="max-height: 300px; overflow-y: auto;">';
|
||||
popupContent += '<table style="width: 100%; border-collapse: collapse;">';
|
||||
|
||||
// Sort properties alphabetically for better organization
|
||||
const sortedKeys = Object.keys(properties).sort();
|
||||
|
||||
for (const key of sortedKeys) {
|
||||
// Skip the label as it's already displayed as the title
|
||||
if (key === 'label') continue;
|
||||
|
||||
const value = properties[key];
|
||||
let displayValue;
|
||||
|
||||
// Format the value based on its type
|
||||
if (value === null || value === undefined) {
|
||||
displayValue = '<em>null</em>';
|
||||
} else if (typeof value === 'object') {
|
||||
displayValue = `<pre style="margin: 0; white-space: pre-wrap;">${JSON.stringify(value, null, 2)}</pre>`;
|
||||
} else if (typeof value === 'string' && value.startsWith('http')) {
|
||||
displayValue = `<a href="${value}" target="_blank">${value}</a>`;
|
||||
} else {
|
||||
displayValue = String(value);
|
||||
}
|
||||
|
||||
popupContent += `
|
||||
<tr style="border-bottom: 1px solid #eee;">
|
||||
<td style="padding: 4px; font-weight: bold; vertical-align: top;">${key}:</td>
|
||||
<td style="padding: 4px;">${displayValue}</td>
|
||||
</tr>`;
|
||||
}
|
||||
|
||||
popupContent += '</table>';
|
||||
popupContent += '</div>';
|
||||
```
|
||||
|
||||
## Benefits
|
||||
|
||||
This enhancement provides several benefits:
|
||||
|
||||
1. **More Information**: Users can now see all available information about an event without having to make additional API calls.
|
||||
2. **Better Debugging**: Developers can more easily debug issues with event data by seeing all properties.
|
||||
3. **Improved User Experience**: The organized layout and smart formatting make it easier to read and understand the event data.
|
||||
4. **Transparency**: Users can see exactly what data is stored for each event.
|
||||
|
||||
## How to Test
|
||||
|
||||
1. Start the server:
|
||||
```bash
|
||||
python3 backend.py
|
||||
```
|
||||
|
||||
2. Open a web browser and navigate to:
|
||||
```
|
||||
http://127.0.0.1:8080/demo
|
||||
```
|
||||
|
||||
3. Click on any event marker on the map to see the enhanced popup with all properties displayed.
|
||||
|
||||
## Future Improvements
|
||||
|
||||
Potential future improvements for the popup display:
|
||||
|
||||
1. Add filtering options to show only certain categories of properties
|
||||
2. Add the ability to copy property values to the clipboard
|
||||
3. Add visualization for temporal data (e.g., timeline for start and stop times)
|
||||
4. Add the ability to edit event properties directly from the popup
|
||||
5. Add support for displaying images or other media that might be included in event properties
|
||||
|
|
@ -161,6 +161,13 @@ dans les extracteurs, vérifier qu'il n'existe pas déjà des évènements avec
|
|||
|
||||
Error: Expecting value: line 1 column 1 (char 0)
|
||||
|
||||
|
||||
-- il manque l'attribution openstreetmap sur les cartes maplibre. ✓
|
||||
ajouter une icone pour ajouter les sources https://source.cipherbliss.com/tykayn/oedb-backend ✓
|
||||
|
||||
ajouter un lien sur la page de démo qui montre comment faire une recherche vers des évènements de type music, et de type sport ✓
|
||||
|
||||
proposer un contrôle de maplibre pour changer de fond de carte openstreetmap vecto ou raster. ✓
|
||||
## License
|
||||
|
||||
See the LICENSE file for details.
|
||||
|
|
|
|||
|
|
@ -1,91 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Script to check if database tables are properly created
|
||||
# Uses environment variables from .env file
|
||||
|
||||
# Load environment variables from .env file
|
||||
if [ -f .env ]; then
|
||||
echo "Loading environment variables from .env file..."
|
||||
export $(grep -v '^#' .env | xargs)
|
||||
fi
|
||||
|
||||
# Default values if not set in .env
|
||||
DB_NAME=${DB_NAME:-"oedb"}
|
||||
DB_USER=${DB_USER:-"postgres"}
|
||||
DB_HOST=${DB_HOST:-"localhost"}
|
||||
|
||||
# Function to check if a table exists
|
||||
check_table() {
|
||||
local table_name=$1
|
||||
echo "Checking if table '$table_name' exists..."
|
||||
|
||||
# Query to check if table exists
|
||||
result=$(PGPASSWORD="$POSTGRES_PASSWORD" psql -h $DB_HOST -U $DB_USER -d $DB_NAME -t -c "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema = 'public' AND table_name = '$table_name');")
|
||||
|
||||
# Trim whitespace
|
||||
result=$(echo $result | xargs)
|
||||
|
||||
if [ "$result" = "t" ]; then
|
||||
echo "✅ Table '$table_name' exists."
|
||||
return 0
|
||||
else
|
||||
echo "❌ Table '$table_name' does not exist!"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to check table structure
|
||||
check_table_columns() {
|
||||
local table_name=$1
|
||||
echo "Checking structure of table '$table_name'..."
|
||||
|
||||
# Get column information
|
||||
PGPASSWORD="$POSTGRES_PASSWORD" psql -h $DB_HOST -U $DB_USER -d $DB_NAME -c "SELECT column_name, data_type FROM information_schema.columns WHERE table_schema = 'public' AND table_name = '$table_name';"
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ Successfully retrieved structure for table '$table_name'."
|
||||
return 0
|
||||
else
|
||||
echo "❌ Failed to retrieve structure for table '$table_name'!"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check database connection
|
||||
echo "Connecting to PostgreSQL database '$DB_NAME' as user '$DB_USER'..."
|
||||
export PGPASSWORD="$POSTGRES_PASSWORD"
|
||||
if ! psql -h $DB_HOST -U $DB_USER -d $DB_NAME -c "SELECT 1" > /dev/null 2>&1; then
|
||||
echo "❌ Failed to connect to database. Please check your credentials and database status."
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ Successfully connected to database."
|
||||
|
||||
# Check required tables
|
||||
tables=("events" "events_deleted" "geo")
|
||||
all_tables_exist=true
|
||||
|
||||
for table in "${tables[@]}"; do
|
||||
if ! check_table "$table"; then
|
||||
all_tables_exist=false
|
||||
fi
|
||||
done
|
||||
|
||||
# If all tables exist, check their structure
|
||||
if [ "$all_tables_exist" = true ]; then
|
||||
echo -e "\n--- Detailed Table Structure ---"
|
||||
for table in "${tables[@]}"; do
|
||||
echo -e "\n"
|
||||
check_table_columns "$table"
|
||||
done
|
||||
|
||||
echo -e "\n✅ All required tables exist in the database."
|
||||
else
|
||||
echo -e "\n❌ Some required tables are missing. Please run setup_db.sh to initialize the database."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check for indexes (optional but recommended)
|
||||
echo -e "\n--- Checking Indexes ---"
|
||||
PGPASSWORD="$POSTGRES_PASSWORD" psql -h $DB_HOST -U $DB_USER -d $DB_NAME -c "SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'public' AND tablename IN ('events', 'events_deleted', 'geo');"
|
||||
|
||||
echo -e "\n✅ Database tables check completed successfully."
|
||||
exit 0
|
||||
|
|
@ -1,2 +1,325 @@
|
|||
# récupérer les évènements depuis osmcal.depuis
|
||||
# https://osmcal.org/events.rss
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
OSM Calendar Extractor for the OpenEventDatabase.
|
||||
|
||||
This script fetches OpenStreetMap events from the osmcal.org RSS feed
|
||||
and adds them to the OpenEventDatabase.
|
||||
|
||||
RSS Feed URL: https://osmcal.org/events.rss
|
||||
"""
|
||||
|
||||
import json
|
||||
import requests
|
||||
import sys
|
||||
import os
|
||||
import feedparser
|
||||
from datetime import datetime, timedelta
|
||||
import pytz
|
||||
from dateutil import parser as date_parser
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# Add the parent directory to the path so we can import from oedb
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
from oedb.utils.db import db_connect
|
||||
from oedb.utils.logging import logger
|
||||
|
||||
# RSS Feed URL for osmcal.org
|
||||
RSS_URL = "https://osmcal.org/events.rss"
|
||||
|
||||
def fetch_osmcal_data():
|
||||
"""
|
||||
Fetch OpenStreetMap events from the osmcal.org RSS feed.
|
||||
|
||||
Returns:
|
||||
list: A list of event entries from the RSS feed.
|
||||
"""
|
||||
logger.info("Fetching data from osmcal.org RSS feed")
|
||||
|
||||
try:
|
||||
# Parse the RSS feed
|
||||
feed = feedparser.parse(RSS_URL)
|
||||
|
||||
if not feed.entries:
|
||||
logger.error("No entries found in RSS feed")
|
||||
return []
|
||||
|
||||
logger.success(f"Successfully fetched {len(feed.entries)} events from osmcal.org")
|
||||
return feed.entries
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching data from osmcal.org: {e}")
|
||||
return []
|
||||
|
||||
def extract_coordinates(location_str):
|
||||
"""
|
||||
Extract coordinates from a location string.
|
||||
|
||||
Args:
|
||||
location_str (str): A string containing location information.
|
||||
|
||||
Returns:
|
||||
tuple: A tuple containing (longitude, latitude) or None if not found.
|
||||
"""
|
||||
# Try to find coordinates in the format "lat,lon" or similar
|
||||
coord_pattern = r'(-?\d+\.\d+)[,\s]+(-?\d+\.\d+)'
|
||||
match = re.search(coord_pattern, location_str)
|
||||
|
||||
if match:
|
||||
lat = float(match.group(1))
|
||||
lon = float(match.group(2))
|
||||
return [lon, lat] # GeoJSON uses [longitude, latitude]
|
||||
|
||||
# Default coordinates (center of France) if none found
|
||||
return [2.2137, 46.2276]
|
||||
|
||||
def parse_date(date_str):
|
||||
"""
|
||||
Parse a date string into an ISO format string.
|
||||
|
||||
Args:
|
||||
date_str (str): A string containing date information.
|
||||
|
||||
Returns:
|
||||
str: An ISO format date string.
|
||||
"""
|
||||
try:
|
||||
# Parse the date string
|
||||
dt = date_parser.parse(date_str)
|
||||
|
||||
# Ensure the datetime is timezone-aware
|
||||
if dt.tzinfo is None:
|
||||
dt = pytz.UTC.localize(dt)
|
||||
|
||||
return dt.isoformat()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing date '{date_str}': {e}")
|
||||
# Return current date as fallback
|
||||
return datetime.now(pytz.UTC).isoformat()
|
||||
|
||||
def create_event(entry):
|
||||
"""
|
||||
Create an event object from an RSS feed entry.
|
||||
|
||||
Args:
|
||||
entry: An entry from the osmcal.org RSS feed.
|
||||
|
||||
Returns:
|
||||
dict: A GeoJSON Feature representing the event.
|
||||
"""
|
||||
try:
|
||||
# Extract data from the entry
|
||||
title = entry.title
|
||||
link = entry.link
|
||||
description = entry.description if hasattr(entry, 'description') else ""
|
||||
|
||||
# Extract dates
|
||||
start_date = None
|
||||
end_date = None
|
||||
|
||||
if hasattr(entry, 'published'):
|
||||
start_date = parse_date(entry.published)
|
||||
|
||||
# If there's no published date, use the current date
|
||||
if not start_date:
|
||||
start_date = datetime.now(pytz.UTC).isoformat()
|
||||
|
||||
# Set end date to 1 day after start date if not specified
|
||||
if not end_date:
|
||||
dt = date_parser.parse(start_date)
|
||||
end_date = (dt + timedelta(days=1)).isoformat()
|
||||
|
||||
# Extract location and coordinates
|
||||
location = ""
|
||||
coordinates = [2.2137, 46.2276] # Default: center of France
|
||||
|
||||
if hasattr(entry, 'where') and entry.where:
|
||||
location = entry.where
|
||||
coordinates = extract_coordinates(location)
|
||||
elif description:
|
||||
# Try to extract location from description
|
||||
location_match = re.search(r'Location:?\s*([^\n]+)', description, re.IGNORECASE)
|
||||
if location_match:
|
||||
location = location_match.group(1).strip()
|
||||
coordinates = extract_coordinates(location)
|
||||
|
||||
# Create a descriptive label
|
||||
label = title
|
||||
|
||||
# Determine the event type
|
||||
what = "community.osm.meetup"
|
||||
|
||||
# Check for specific event types in the title or description
|
||||
lower_title = title.lower()
|
||||
lower_desc = description.lower()
|
||||
|
||||
if any(term in lower_title or term in lower_desc for term in ["conference", "summit"]):
|
||||
what = "community.osm.conference"
|
||||
elif any(term in lower_title or term in lower_desc for term in ["workshop", "training"]):
|
||||
what = "community.osm.workshop"
|
||||
elif any(term in lower_title or term in lower_desc for term in ["mapathon", "mapping party"]):
|
||||
what = "community.osm.mapathon"
|
||||
|
||||
# Create the event object
|
||||
event = {
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": coordinates
|
||||
},
|
||||
"properties": {
|
||||
"type": "scheduled",
|
||||
"what": what,
|
||||
"what:series": "OpenStreetMap Events",
|
||||
"where": location,
|
||||
"label": label,
|
||||
"description": description,
|
||||
"start": start_date,
|
||||
"stop": end_date,
|
||||
"url": link,
|
||||
"source": "osmcal.org"
|
||||
}
|
||||
}
|
||||
|
||||
return event
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating event from entry: {e}")
|
||||
return None
|
||||
|
||||
def submit_event(event):
|
||||
"""
|
||||
Submit an event to the OpenEventDatabase.
|
||||
|
||||
Args:
|
||||
event: A GeoJSON Feature representing the event.
|
||||
|
||||
Returns:
|
||||
bool: True if the event was successfully submitted, False otherwise.
|
||||
"""
|
||||
try:
|
||||
# Connect to the database
|
||||
db = db_connect()
|
||||
cur = db.cursor()
|
||||
|
||||
# Extract event properties
|
||||
properties = event['properties']
|
||||
geometry = json.dumps(event['geometry'])
|
||||
|
||||
# Insert the geometry into the geo table
|
||||
cur.execute("""
|
||||
INSERT INTO geo
|
||||
SELECT geom, md5(st_astext(geom)) as hash, st_centroid(geom) as geom_center FROM
|
||||
(SELECT st_setsrid(st_geomfromgeojson(%s),4326) as geom) as g
|
||||
WHERE ST_IsValid(geom)
|
||||
ON CONFLICT DO NOTHING RETURNING hash;
|
||||
""", (geometry,))
|
||||
|
||||
# Get the geometry hash
|
||||
hash_result = cur.fetchone()
|
||||
|
||||
if hash_result is None:
|
||||
# If the hash is None, get it from the database
|
||||
cur.execute("""
|
||||
SELECT md5(st_asewkt(geom)),
|
||||
ST_IsValid(geom),
|
||||
ST_IsValidReason(geom) from (SELECT st_geomfromgeojson(%s) as geom) as g;
|
||||
""", (geometry,))
|
||||
hash_result = cur.fetchone()
|
||||
|
||||
if hash_result is None or (len(hash_result) > 1 and not hash_result[1]):
|
||||
logger.error(f"Invalid geometry for event: {properties.get('label')}")
|
||||
db.close()
|
||||
return False
|
||||
|
||||
geo_hash = hash_result[0]
|
||||
|
||||
# Determine the bounds for the time range
|
||||
bounds = '[]' if properties['start'] == properties['stop'] else '[)'
|
||||
|
||||
# Check if an event with the same properties already exists
|
||||
cur.execute("""
|
||||
SELECT events_id FROM events
|
||||
WHERE events_what = %s
|
||||
AND events_when = tstzrange(%s, %s, %s)
|
||||
AND events_geo = %s;
|
||||
""", (
|
||||
properties['what'],
|
||||
properties['start'],
|
||||
properties['stop'],
|
||||
bounds,
|
||||
geo_hash
|
||||
))
|
||||
|
||||
existing_id = cur.fetchone()
|
||||
|
||||
if existing_id:
|
||||
logger.info(f"Event already exists with ID: {existing_id[0]}")
|
||||
db.close()
|
||||
return False
|
||||
|
||||
# Insert the event into the database
|
||||
cur.execute("""
|
||||
INSERT INTO events (events_type, events_what, events_when, events_tags, events_geo)
|
||||
VALUES (%s, %s, tstzrange(%s, %s, %s), %s, %s)
|
||||
ON CONFLICT DO NOTHING RETURNING events_id;
|
||||
""", (
|
||||
properties['type'],
|
||||
properties['what'],
|
||||
properties['start'],
|
||||
properties['stop'],
|
||||
bounds,
|
||||
json.dumps(properties),
|
||||
geo_hash
|
||||
))
|
||||
|
||||
# Get the event ID
|
||||
event_id = cur.fetchone()
|
||||
|
||||
if event_id:
|
||||
logger.success(f"Event created with ID: {event_id[0]}")
|
||||
db.commit()
|
||||
db.close()
|
||||
return True
|
||||
else:
|
||||
logger.warning(f"Failed to create event: {properties.get('label')}")
|
||||
db.close()
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error submitting event: {e}")
|
||||
return False
|
||||
|
||||
def main():
|
||||
"""
|
||||
Main function to fetch OSM Calendar data and add events to the database.
|
||||
"""
|
||||
logger.info("Starting OSM Calendar extractor")
|
||||
|
||||
# Fetch data from osmcal.org
|
||||
entries = fetch_osmcal_data()
|
||||
|
||||
if not entries:
|
||||
logger.warning("No entries found, exiting")
|
||||
return
|
||||
|
||||
# Process each entry
|
||||
success_count = 0
|
||||
|
||||
for entry in entries:
|
||||
# Create an event from the entry
|
||||
event = create_event(entry)
|
||||
|
||||
if not event:
|
||||
continue
|
||||
|
||||
# Submit the event to the database
|
||||
if submit_event(event):
|
||||
success_count += 1
|
||||
|
||||
logger.success(f"Successfully added {success_count} events to the database")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -163,6 +163,9 @@ class DemoResource:
|
|||
<a href="/demo">← Back to Map</a>
|
||||
<a href="/">API Information</a>
|
||||
<a href="/event">View Events</a>
|
||||
<a href="https://source.cipherbliss.com/tykayn/oedb-backend" target="_blank" title="View Source Code on Cipherbliss">
|
||||
<i class="fas fa-code-branch"></i> Source
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<h1>Edit Event</h1>
|
||||
|
|
@ -242,6 +245,11 @@ class DemoResource:
|
|||
// Add navigation controls
|
||||
map.addControl(new maplibregl.NavigationControl());
|
||||
|
||||
// Add attribution control with OpenStreetMap attribution
|
||||
map.addControl(new maplibregl.AttributionControl({{
|
||||
customAttribution: '© <a href="https://www.openstreetmap.org/copyright" target="_blank">OpenStreetMap</a> contributors'
|
||||
}}));
|
||||
|
||||
// Add marker for event location
|
||||
let marker = new maplibregl.Marker({{
|
||||
draggable: true
|
||||
|
|
@ -448,6 +456,35 @@ class DemoResource:
|
|||
box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
|
||||
max-width: 300px;
|
||||
}
|
||||
.map-style-control {
|
||||
position: absolute;
|
||||
top: 10px;
|
||||
right: 10px;
|
||||
background: rgba(255, 255, 255, 0.9);
|
||||
padding: 10px;
|
||||
border-radius: 5px;
|
||||
box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
|
||||
z-index: 1;
|
||||
}
|
||||
.map-style-control button {
|
||||
display: block;
|
||||
margin-bottom: 5px;
|
||||
padding: 5px 10px;
|
||||
background: #fff;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 3px;
|
||||
cursor: pointer;
|
||||
width: 100%;
|
||||
text-align: left;
|
||||
}
|
||||
.map-style-control button:hover {
|
||||
background: #f5f5f5;
|
||||
}
|
||||
.map-style-control button.active {
|
||||
background: #0078ff;
|
||||
color: white;
|
||||
border-color: #0056b3;
|
||||
}
|
||||
h2 { margin-top: 0; }
|
||||
ul { padding-left: 20px; }
|
||||
a { color: #0078ff; text-decoration: none; }
|
||||
|
|
@ -470,16 +507,56 @@ class DemoResource:
|
|||
<ul>
|
||||
<li><a href="/demo/by-what" target="_blank">/demo/by-what - Events by Type</a></li>
|
||||
<li><a href="/demo/map-by-what" target="_blank">/demo/map-by-what - Map by Event Type</a></li>
|
||||
<li><a href="/event?what=music" target="_blank">Search Music Events</a></li>
|
||||
<li><a href="/event?what=sport" target="_blank">Search Sport Events</a></li>
|
||||
</ul>
|
||||
<p><a href="/demo/add" class="add-event-btn" style="display: block; text-align: center; margin-top: 15px; padding: 8px; background-color: #0078ff; color: white; border-radius: 4px; font-weight: bold;">+ Add New Event</a></p>
|
||||
<p><a href="https://source.cipherbliss.com/tykayn/oedb-backend" target="_blank">Source Code on Cipherbliss</a></p>
|
||||
<p style="text-align: center; margin-top: 10px;">
|
||||
<a href="https://source.cipherbliss.com/tykayn/oedb-backend" target="_blank" title="View Source Code on Cipherbliss" style="font-size: 24px;">
|
||||
<i class="fas fa-code-branch"></i>
|
||||
</a>
|
||||
</p>
|
||||
</div>
|
||||
<div class="map-style-control">
|
||||
<h4 style="margin-top: 0; margin-bottom: 10px;">Map Style</h4>
|
||||
<button id="style-default" class="active">Default</button>
|
||||
<button id="style-osm-vector">OSM Vector</button>
|
||||
<button id="style-osm-raster">OSM Raster</button>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Initialize the map
|
||||
// Map style URLs
|
||||
const mapStyles = {
|
||||
default: 'https://demotiles.maplibre.org/style.json',
|
||||
osmVector: 'https://cdn.jsdelivr.net/gh/openmaptiles/osm-bright-gl-style@master/style-cdn.json',
|
||||
osmRaster: {
|
||||
version: 8,
|
||||
sources: {
|
||||
'osm-raster': {
|
||||
type: 'raster',
|
||||
tiles: [
|
||||
'https://tile.openstreetmap.org/{z}/{x}/{y}.png'
|
||||
],
|
||||
tileSize: 256,
|
||||
attribution: '© <a href="https://www.openstreetmap.org/copyright" target="_blank">OpenStreetMap</a> contributors'
|
||||
}
|
||||
},
|
||||
layers: [
|
||||
{
|
||||
id: 'osm-raster-layer',
|
||||
type: 'raster',
|
||||
source: 'osm-raster',
|
||||
minzoom: 0,
|
||||
maxzoom: 19
|
||||
}
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize the map with default style
|
||||
const map = new maplibregl.Map({
|
||||
container: 'map',
|
||||
style: 'https://demotiles.maplibre.org/style.json',
|
||||
style: mapStyles.default,
|
||||
center: [2.3522, 48.8566], // Default center (Paris)
|
||||
zoom: 4
|
||||
});
|
||||
|
|
@ -487,6 +564,72 @@ class DemoResource:
|
|||
// Add navigation controls
|
||||
map.addControl(new maplibregl.NavigationControl());
|
||||
|
||||
// Add attribution control with OpenStreetMap attribution
|
||||
map.addControl(new maplibregl.AttributionControl({
|
||||
customAttribution: '© <a href="https://www.openstreetmap.org/copyright" target="_blank">OpenStreetMap</a> contributors'
|
||||
}));
|
||||
|
||||
// Style switcher functionality
|
||||
let currentStyle = 'default';
|
||||
let eventsData = null;
|
||||
|
||||
document.getElementById('style-default').addEventListener('click', () => {
|
||||
if (currentStyle !== 'default') {
|
||||
switchMapStyle('default');
|
||||
}
|
||||
});
|
||||
|
||||
document.getElementById('style-osm-vector').addEventListener('click', () => {
|
||||
if (currentStyle !== 'osmVector') {
|
||||
switchMapStyle('osmVector');
|
||||
}
|
||||
});
|
||||
|
||||
document.getElementById('style-osm-raster').addEventListener('click', () => {
|
||||
if (currentStyle !== 'osmRaster') {
|
||||
switchMapStyle('osmRaster');
|
||||
}
|
||||
});
|
||||
|
||||
function switchMapStyle(styleName) {
|
||||
// Update active button
|
||||
document.querySelectorAll('.map-style-control button').forEach(button => {
|
||||
button.classList.remove('active');
|
||||
});
|
||||
document.getElementById(`style-${styleName.replace('osm', 'osm-')}`).classList.add('active');
|
||||
|
||||
// Save current center and zoom
|
||||
const center = map.getCenter();
|
||||
const zoom = map.getZoom();
|
||||
|
||||
// Save events data if already loaded
|
||||
if (map.getSource('events')) {
|
||||
try {
|
||||
eventsData = map.getSource('events')._data;
|
||||
} catch (e) {
|
||||
console.error('Error saving events data:', e);
|
||||
}
|
||||
}
|
||||
|
||||
// Apply new style
|
||||
map.setStyle(mapStyles[styleName]);
|
||||
|
||||
// Restore center and zoom after style is loaded
|
||||
map.once('style.load', () => {
|
||||
map.setCenter(center);
|
||||
map.setZoom(zoom);
|
||||
|
||||
// Restore events data if available
|
||||
if (eventsData) {
|
||||
addEventsToMap(eventsData);
|
||||
} else {
|
||||
fetchEvents();
|
||||
}
|
||||
});
|
||||
|
||||
currentStyle = styleName;
|
||||
}
|
||||
|
||||
// Fetch events when the map is loaded
|
||||
map.on('load', function() {
|
||||
fetchEvents();
|
||||
|
|
@ -688,8 +831,19 @@ class DemoResource:
|
|||
resp.content_type = 'text/html'
|
||||
|
||||
# Fetch events from the API
|
||||
response = requests.get('http://localhost/event?limit=1000')
|
||||
events_data = response.json()
|
||||
try:
|
||||
response = requests.get('/event?limit=1000')
|
||||
if response.status_code == 200 and response.text:
|
||||
events_data = response.json()
|
||||
else:
|
||||
logger.error(f"Error fetching events: Status code {response.status_code}, Response: {response.text}")
|
||||
events_data = {"features": []}
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Error parsing JSON response: {e}")
|
||||
events_data = {"features": []}
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching events: {e}")
|
||||
events_data = {"features": []}
|
||||
|
||||
# Group events by "what" type
|
||||
events_by_what = defaultdict(list)
|
||||
|
|
@ -933,6 +1087,9 @@ class DemoResource:
|
|||
<a href="/">Home</a>
|
||||
<a href="/demo">Demo Map</a>
|
||||
<a href="/demo/by-what">Events by Type</a>
|
||||
<a href="https://source.cipherbliss.com/tykayn/oedb-backend" target="_blank" title="View Source Code on Cipherbliss">
|
||||
<i class="fas fa-code-branch"></i> Source
|
||||
</a>
|
||||
</div>
|
||||
<p>This map shows events from the OpenEventDatabase filtered by their type.</p>
|
||||
<p>Use the filter panel on the right to show/hide different event types.</p>
|
||||
|
|
@ -964,6 +1121,11 @@ class DemoResource:
|
|||
// Add navigation controls
|
||||
map.addControl(new maplibregl.NavigationControl());
|
||||
|
||||
// Add attribution control with OpenStreetMap attribution
|
||||
map.addControl(new maplibregl.AttributionControl({
|
||||
customAttribution: '© <a href="https://www.openstreetmap.org/copyright" target="_blank">OpenStreetMap</a> contributors'
|
||||
}));
|
||||
|
||||
// Store all events and their types
|
||||
let allEvents = null;
|
||||
let eventTypes = new Set();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue