From 153e9bd8f9e7b626e1551bd8cae5f36f756aa93e Mon Sep 17 00:00:00 2001
From: Tykayn <contact@cipherbliss.com>
Date: Thu, 18 Sep 2025 22:30:25 +0200
Subject: [PATCH] up extractor env getting

---
 doc/SEARCH_ENDPOINT.md                        |   4 +-
 doc/{ANTI_SPAM_MEASURES.md => anti_spam.md}   |   0
 doc/{TEST_PLAN.md => api_load_test.md}        |   0
 ...PI_QUERY_PARAMS.md => api_query_params.md} |   0
 doc/database_schema.svg                       |   3 +
 doc/{DEMO_ENDPOINT.md => demo_endpoint.md}    |   0
 extractors/osm_cal.py                         |  11 +-
 oedb/resources/stats.py                       |   7 ++
 oedb/utils/db.py                              |   7 ++
 test_env_file_check.py                        | 117 ++++++++++++++++++
 test_stats_endpoint.py                        |  51 ++++++++
 11 files changed, 196 insertions(+), 4 deletions(-)
 rename doc/{ANTI_SPAM_MEASURES.md => anti_spam.md} (100%)
 rename doc/{TEST_PLAN.md => api_load_test.md} (100%)
 rename doc/{API_QUERY_PARAMS.md => api_query_params.md} (100%)
 rename doc/{DEMO_ENDPOINT.md => demo_endpoint.md} (100%)
 create mode 100755 test_env_file_check.py
 create mode 100755 test_stats_endpoint.py

diff --git a/doc/SEARCH_ENDPOINT.md b/doc/SEARCH_ENDPOINT.md
index d6e4a76..ca8aaf0 100644
--- a/doc/SEARCH_ENDPOINT.md
+++ b/doc/SEARCH_ENDPOINT.md
@@ -39,7 +39,7 @@ Example request body:
 
 The search endpoint supports all the same query parameters as the `/event` endpoint. These parameters can be used to further refine your search beyond the geometric constraints.
 
-For detailed information about the available query parameters, see the [API Query Parameters Documentation](API_QUERY_PARAMS.md).
+For detailed information about the available query parameters, see the [API Query Parameters Documentation](api_query_params.md).
 
 ### Additional Parameters
 
@@ -174,7 +174,7 @@ The OpenEventDatabase API implements caching and rate limiting to improve perfor
 - POST requests to `/event/search` are limited to 20 requests per minute per IP address.
 - If you exceed this limit, you will receive a `429 Too Many Requests` response with a `Retry-After` header indicating when you can try again.
 
-For more information about the anti-spam and caching measures implemented in the API, see the [Anti-Spam and Caching Measures Documentation](ANTI_SPAM_MEASURES.md).
+For more information about the anti-spam and caching measures implemented in the API, see the [Anti-Spam and Caching Measures Documentation](anti_spam.md).
 
 ## Error Handling
 
diff --git a/doc/ANTI_SPAM_MEASURES.md b/doc/anti_spam.md
similarity index 100%
rename from doc/ANTI_SPAM_MEASURES.md
rename to doc/anti_spam.md
diff --git a/doc/TEST_PLAN.md b/doc/api_load_test.md
similarity index 100%
rename from doc/TEST_PLAN.md
rename to doc/api_load_test.md
diff --git a/doc/API_QUERY_PARAMS.md b/doc/api_query_params.md
similarity index 100%
rename from doc/API_QUERY_PARAMS.md
rename to doc/api_query_params.md
diff --git a/doc/database_schema.svg b/doc/database_schema.svg
index 1466a59..e8aa5b0 100644
--- a/doc/database_schema.svg
+++ b/doc/database_schema.svg
@@ -1,6 +1,9 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <svg width="800" height="600" xmlns="http://www.w3.org/2000/svg">
   <style>
+    svg{
+    background : white;
+    }
     .table {
       fill: #f5f5f5;
       stroke: #333;
diff --git a/doc/DEMO_ENDPOINT.md b/doc/demo_endpoint.md
similarity index 100%
rename from doc/DEMO_ENDPOINT.md
rename to doc/demo_endpoint.md
diff --git a/extractors/osm_cal.py b/extractors/osm_cal.py
index 79a8c36..83bce59 100755
--- a/extractors/osm_cal.py
+++ b/extractors/osm_cal.py
@@ -414,11 +414,18 @@ def submit_event(event):
 def main():
     """
     Main function to fetch OSM Calendar events and add them to the database.
+    
+    The function will exit if the .env file doesn't exist, as it's required
+    for database connection parameters.
     """
     logger.info("Starting OSM Calendar extractor")
     
-    # Load environment variables from .env file if it exists
-    load_env_from_file()
+    # Load environment variables from .env file and check if it exists
+    if not load_env_from_file():
+        logger.error("Required .env file not found. Exiting.")
+        sys.exit(1)
+        
+    logger.info("Environment variables loaded successfully from .env file")
 
     # Fetch events from the OSM Calendar RSS feed
     items = fetch_osm_calendar_data()
diff --git a/oedb/resources/stats.py b/oedb/resources/stats.py
index c44dcda..d149ea5 100644
--- a/oedb/resources/stats.py
+++ b/oedb/resources/stats.py
@@ -34,6 +34,13 @@ class StatsResource:
             count = cur.fetchone()[0]
             logger.debug(f"Estimated event count: {count}")
             
+            # If the estimated count is negative, fall back to an actual count
+            if count < 0:
+                logger.warning(f"Estimated event count is negative ({count}), falling back to COUNT(*)")
+                cur.execute("SELECT COUNT(*) FROM events;")
+                count = cur.fetchone()[0]
+                logger.debug(f"Actual event count: {count}")
+            
             # Global info
             cur.execute("SELECT max(lastupdate) as last_updated, current_timestamp-pg_postmaster_start_time() from events;")
             pg_stats = cur.fetchone()
diff --git a/oedb/utils/db.py b/oedb/utils/db.py
index e99d3a3..5acf7e9 100644
--- a/oedb/utils/db.py
+++ b/oedb/utils/db.py
@@ -12,6 +12,9 @@ def load_env_from_file():
     """
     Load environment variables from .env file.
     This ensures that database connection parameters are properly set.
+    
+    Returns:
+        bool: True if the .env file exists and was loaded, False otherwise.
     """
     if os.path.exists('.env'):
         logger.info("Loading environment variables from .env file...")
@@ -21,6 +24,10 @@ def load_env_from_file():
                 if line and not line.startswith('#'):
                     key, value = line.split('=', 1)
                     os.environ[key] = value
+        return True
+    else:
+        logger.warning(".env file not found")
+        return False
 
 def db_connect():
     """
diff --git a/test_env_file_check.py b/test_env_file_check.py
new file mode 100755
index 0000000..91185ee
--- /dev/null
+++ b/test_env_file_check.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""
+Test script to verify the behavior of the load_env_from_file function
+and the osm_cal.py script when the .env file exists or doesn't exist.
+"""
+
+import os
+import sys
+import subprocess
+import shutil
+from oedb.utils.db import load_env_from_file
+
+def test_load_env_from_file():
+    """
+    Test the load_env_from_file function directly.
+    """
+    print("\n=== Testing load_env_from_file function ===")
+    
+    # Backup the .env file if it exists
+    env_exists = os.path.exists('.env')
+    if env_exists:
+        print("Backing up existing .env file...")
+        shutil.copy('.env', '.env.backup')
+    
+    # Test when .env file doesn't exist
+    if env_exists:
+        os.remove('.env')
+    
+    print("\nTesting when .env file doesn't exist:")
+    result = load_env_from_file()
+    print(f"load_env_from_file() returned: {result}")
+    
+    # Create a test .env file
+    print("\nCreating test .env file...")
+    with open('.env', 'w') as f:
+        f.write("TEST_VAR=test_value\n")
+    
+    # Test when .env file exists
+    print("\nTesting when .env file exists:")
+    result = load_env_from_file()
+    print(f"load_env_from_file() returned: {result}")
+    print(f"TEST_VAR environment variable: {os.getenv('TEST_VAR')}")
+    
+    # Clean up
+    os.remove('.env')
+    
+    # Restore the original .env file if it existed
+    if env_exists:
+        print("\nRestoring original .env file...")
+        shutil.move('.env.backup', '.env')
+
+def test_osm_cal_script():
+    """
+    Test the osm_cal.py script behavior with and without .env file.
+    """
+    print("\n=== Testing osm_cal.py script ===")
+    
+    # Backup the .env file if it exists
+    env_exists = os.path.exists('.env')
+    if env_exists:
+        print("Backing up existing .env file...")
+        shutil.copy('.env', '.env.backup')
+    
+    # Test when .env file doesn't exist
+    if env_exists:
+        os.remove('.env')
+    
+    print("\nTesting osm_cal.py when .env file doesn't exist:")
+    try:
+        result = subprocess.run(
+            [sys.executable, 'extractors/osm_cal.py'],
+            capture_output=True,
+            text=True,
+            timeout=5  # Timeout after 5 seconds
+        )
+        print(f"Exit code: {result.returncode}")
+        print(f"Output: {result.stdout}")
+        print(f"Error: {result.stderr}")
+    except subprocess.TimeoutExpired:
+        print("Process timed out - this might indicate it's trying to fetch RSS feed despite missing .env")
+    
+    # Create a test .env file with minimal required variables
+    print("\nCreating test .env file...")
+    with open('.env', 'w') as f:
+        f.write("DB_NAME=test_db\n")
+        f.write("DB_HOST=localhost\n")
+        f.write("DB_USER=test_user\n")
+        f.write("POSTGRES_PASSWORD=test_password\n")
+    
+    # Test when .env file exists
+    print("\nTesting osm_cal.py when .env file exists:")
+    try:
+        # We'll use a very short timeout since we expect it to try to connect to the database
+        # and fail, but we just want to verify it gets past the .env check
+        result = subprocess.run(
+            [sys.executable, 'extractors/osm_cal.py'],
+            capture_output=True,
+            text=True,
+            timeout=2  # Short timeout
+        )
+        print(f"Exit code: {result.returncode}")
+        print(f"Output: {result.stdout}")
+        print(f"Error: {result.stderr}")
+    except subprocess.TimeoutExpired:
+        print("Process timed out - this is expected as it's trying to connect to the database")
+    
+    # Clean up
+    os.remove('.env')
+    
+    # Restore the original .env file if it existed
+    if env_exists:
+        print("\nRestoring original .env file...")
+        shutil.move('.env.backup', '.env')
+
+if __name__ == "__main__":
+    test_load_env_from_file()
+    test_osm_cal_script()
\ No newline at end of file
diff --git a/test_stats_endpoint.py b/test_stats_endpoint.py
new file mode 100755
index 0000000..65fecb7
--- /dev/null
+++ b/test_stats_endpoint.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""
+Test script to check the stats endpoint response.
+"""
+
+import requests
+import json
+import sys
+
+# URL of the stats endpoint (adjust if needed)
+STATS_URL = "http://localhost:8080/stats"
+
+def test_stats_endpoint():
+    """
+    Make a request to the stats endpoint and print the response.
+    """
+    try:
+        print(f"Making request to {STATS_URL}...")
+        response = requests.get(STATS_URL)
+        
+        # Check if the request was successful
+        if response.status_code == 200:
+            print("Request successful!")
+            data = response.json()
+            
+            # Print the events_count value
+            print(f"events_count: {data.get('events_count')}")
+            
+            # Print other relevant information
+            print(f"last_updated: {data.get('last_updated')}")
+            print(f"uptime: {data.get('uptime')}")
+            print(f"db_uptime: {data.get('db_uptime')}")
+            
+            # Check if events_count is -1
+            if data.get('events_count') == -1:
+                print("ISSUE CONFIRMED: events_count is -1")
+            else:
+                print(f"events_count is {data.get('events_count')}")
+                
+            return data
+        else:
+            print(f"Request failed with status code: {response.status_code}")
+            print(f"Response: {response.text}")
+            return None
+            
+    except Exception as e:
+        print(f"Error making request: {e}")
+        return None
+
+if __name__ == "__main__":
+    test_stats_endpoint()
\ No newline at end of file