commande pour créer des stats de toutes les villes insee

# Conflicts:
#	src/Command/ProcessLabourageQueueCommand.php
#	src/Controller/PublicController.php
This commit is contained in:
Tykayn 2025-08-08 18:09:24 +02:00 committed by tykayn
parent dfeaf123f4
commit 8cfea30fdf
6 changed files with 475 additions and 44 deletions

View file

@ -0,0 +1,194 @@
<?php
namespace App\Command;
use App\Entity\CityFollowUp;
use App\Entity\Stats;
use App\Repository\StatsRepository;
use Doctrine\ORM\EntityManagerInterface;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
#[AsCommand(
name: 'app:create-missing-communes-stats',
description: 'Create Stats objects for missing communes using data from cities_insee.csv',
)]
class CreateMissingCommunesStatsCommand extends Command
{
private const CSV_PATH = '/home/poule/encrypted/stockage-syncable/www/development/html/osm-commerce-sf/counting_osm_objects/cities_insee.csv';
// List of themes to create CityFollowUp measurements for
private const THEMES = [
'charging_station',
'defibrillator',
'shop',
'amenity',
// Add more themes as needed
];
private EntityManagerInterface $entityManager;
private StatsRepository $statsRepository;
public function __construct(
EntityManagerInterface $entityManager,
StatsRepository $statsRepository
) {
parent::__construct();
$this->entityManager = $entityManager;
$this->statsRepository = $statsRepository;
}
protected function configure(): void
{
$this
->addOption('limit', 'l', InputOption::VALUE_REQUIRED, 'Limit the number of communes to process', 0)
->addOption('dry-run', null, InputOption::VALUE_NONE, 'Simulate without modifying the database');
}
protected function execute(InputInterface $input, OutputInterface $output): int
{
$io = new SymfonyStyle($input, $output);
$io->title('Creating Stats objects for missing communes');
$limit = (int) $input->getOption('limit');
$dryRun = $input->getOption('dry-run');
// Check if CSV file exists
if (!file_exists(self::CSV_PATH)) {
$io->error('CSV file not found: ' . self::CSV_PATH);
return Command::FAILURE;
}
// Read CSV file
$io->info('Reading CSV file: ' . self::CSV_PATH);
$communes = $this->readCsvFile(self::CSV_PATH);
$io->info(sprintf('Found %d communes in CSV file', count($communes)));
// Get existing Stats objects
$existingStats = $this->statsRepository->findAll();
$existingZones = [];
foreach ($existingStats as $stats) {
$existingZones[$stats->getZone()] = true;
}
$io->info(sprintf('Found %d existing Stats objects', count($existingZones)));
// Find missing communes
$missingCommunes = [];
foreach ($communes as $commune) {
if (!isset($existingZones[$commune['code_insee']])) {
$missingCommunes[] = $commune;
}
}
$io->info(sprintf('Found %d missing communes', count($missingCommunes)));
// Apply limit if specified
if ($limit > 0 && count($missingCommunes) > $limit) {
$io->info(sprintf('Limiting to %d communes', $limit));
$missingCommunes = array_slice($missingCommunes, 0, $limit);
}
// Create Stats objects for missing communes
$created = 0;
$now = new \DateTime();
foreach ($missingCommunes as $commune) {
$io->text(sprintf('Processing commune: %s (%s)', $commune['nom_standard'], $commune['code_insee']));
if (!$dryRun) {
// Create new Stats object
$stats = new Stats();
$stats->setZone($commune['code_insee']);
$stats->setName($commune['nom_standard']);
// Handle population - convert empty string to null or cast to int
$population = $commune['population'] !== '' ? (int) $commune['population'] : null;
$stats->setPopulation($population);
$stats->setDateCreated($now);
$stats->setKind('command'); // Set the kind to 'command'
// Set coordinates if available and not empty
if (isset($commune['latitude_centre']) && isset($commune['longitude_centre'])) {
// Convert empty strings to null for numeric fields
$lat = $commune['latitude_centre'] !== '' ? $commune['latitude_centre'] : null;
$lon = $commune['longitude_centre'] !== '' ? $commune['longitude_centre'] : null;
$stats->setLat($lat);
$stats->setLon($lon);
}
// Create CityFollowUp measurements for each theme
foreach (self::THEMES as $theme) {
// Create a basic measurement with a default value
// In a real scenario, you would fetch actual data for each theme
$followUp = new CityFollowUp();
$followUp->setName($theme . '_count');
$followUp->setMeasure(0); // Default value, should be replaced with actual data
$followUp->setDate($now);
$followUp->setStats($stats);
$this->entityManager->persist($followUp);
}
$this->entityManager->persist($stats);
$created++;
// Flush every 20 entities to avoid memory issues
if ($created % 20 === 0) {
$this->entityManager->flush();
$this->entityManager->clear(Stats::class);
$this->entityManager->clear(CityFollowUp::class);
$io->text(sprintf('Flushed after creating %d Stats objects', $created));
}
} else {
$created++;
}
}
// Final flush
if (!$dryRun && $created > 0) {
$this->entityManager->flush();
}
if ($dryRun) {
$io->success(sprintf('Dry run completed. Would have created %d Stats objects for missing communes.', $created));
} else {
$io->success(sprintf('Created %d Stats objects for missing communes.', $created));
}
return Command::SUCCESS;
}
/**
* Read CSV file and return an array of communes
*/
private function readCsvFile(string $path): array
{
$communes = [];
if (($handle = fopen($path, 'r')) !== false) {
// Read header
$header = fgetcsv($handle, 0, ',');
if ($header === false) {
return [];
}
// Read data
while (($row = fgetcsv($handle, 0, ',')) !== false) {
$commune = [];
foreach ($header as $i => $key) {
$commune[$key] = $row[$i] ?? '';
}
$communes[] = $commune;
}
fclose($handle);
}
return $communes;
}
}

View file

@ -95,6 +95,7 @@ class CreateStatsFromDemandesCommand extends Command
$stats->setDateCreated(new \DateTime());
$stats->setDateLabourageRequested(new \DateTime());
$stats->setKind('request'); // Set the kind to 'request' as it's created from Demandes
$this->entityManager->persist($stats);
$newStatsCount++;

View file

@ -2,16 +2,16 @@
namespace App\Command;
use App\Entity\Stats;
use App\Entity\Place;
use App\Entity\Stats;
use App\Service\FollowUpService;
use App\Service\Motocultrice;
use Doctrine\ORM\EntityManagerInterface;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use App\Service\Motocultrice;
use App\Service\FollowUpService;
#[AsCommand(
name: 'app:process-labourage-queue',
@ -21,9 +21,10 @@ class ProcessLabourageQueueCommand extends Command
{
public function __construct(
private EntityManagerInterface $entityManager,
private Motocultrice $motocultrice,
private FollowUpService $followUpService
) {
private Motocultrice $motocultrice,
private FollowUpService $followUpService
)
{
parent::__construct();
}
@ -142,26 +143,36 @@ class ProcessLabourageQueueCommand extends Command
}
}
$stats->setDateLabourageDone(new \DateTime());
// Update city name from API if available
$apiCityName = $this->motocultrice->get_city_osm_from_zip_code($stats->getZone());
if ($apiCityName && $apiCityName !== $stats->getName()) {
$io->info(sprintf('Updating city name from "%s" to "%s" based on API data', $stats->getName(), $apiCityName));
$stats->setName($apiCityName);
}
$io->info('Récupération des followups de cette ville...');
// $this->followUpService->generateCityFollowUps($stats, $this->motocultrice, $this->entityManager);
// update completion
$stats->computeCompletionPercent();
$followups = $stats->getCityFollowUps();
if ($followups) {
$lastFollowUp = $followups[count($followups) - 1];
if ($lastFollowUp) {
$name = $lastFollowUp->getName();
$io->success("Followup le plus récent : $name : " . $lastFollowUp->getDate()->format('d/m/Y') . ' : ' . $lastFollowUp->getMeasure());
}
}
$io->info('Pourcentage de complétion: ' . $stats->getCompletionPercent());
$this->entityManager->persist($stats);
$this->entityManager->flush();
$io->success("Labourage terminé : $processedCount nouveaux lieux, $updatedCount lieux mis à jour.");
return Command::SUCCESS;
}

View file

@ -771,6 +771,7 @@ final class AdminController extends AbstractController
if (!$stats) {
$stats = new Stats();
$stats->setZone($insee_code);
$stats->setKind('request'); // Set the kind to 'request' as it's created from an admin request
// $this->addFlash('error', '3 Aucune stats trouvée pour ce code INSEE.');
// return $this->redirectToRoute('app_public_index');
}
@ -1337,7 +1338,8 @@ final class AdminController extends AbstractController
$stats->setZone($zone)
->setName($name)
->setDateCreated(new \DateTime())
->setDateModified(new \DateTime());
->setDateModified(new \DateTime())
->setKind('request'); // Set the kind to 'request' as it's created from an admin import
// Remplir les champs optionnels
if (isset($statData['population'])) {

View file

@ -122,6 +122,138 @@ class PublicController extends AbstractController
return $this->redirectToRoute('app_public_index');
}
#[Route('/api/demande/create', name: 'app_public_create_demande', methods: ['POST'])]
public function createDemande(Request $request): JsonResponse
{
$data = json_decode($request->getContent(), true);
if (!isset($data['businessName']) || empty($data['businessName'])) {
return new JsonResponse(['success' => false, 'message' => 'Le nom du commerce est requis'], 400);
}
// Create a new Demande
$demande = new Demande();
$demande->setQuery($data['businessName']);
$demande->setStatus('new');
$demande->setCreatedAt(new \DateTime());
// Save the INSEE code if provided
if (isset($data['insee']) && !empty($data['insee'])) {
$demande->setInsee((int)$data['insee']);
}
// Save the OSM object type if provided
if (isset($data['osmObjectType']) && !empty($data['osmObjectType'])) {
$demande->setOsmObjectType($data['osmObjectType']);
}
// Save the OSM ID if provided
if (isset($data['osmId']) && !empty($data['osmId'])) {
$demande->setOsmId($data['osmId']);
}
// Check if a Place exists with the same OSM ID and type
$place = null;
if ($demande->getOsmId() && $demande->getOsmObjectType()) {
$existingPlace = $this->entityManager->getRepository(Place::class)->findOneBy([
'osm_kind' => $demande->getOsmObjectType(),
'osmId' => $demande->getOsmId()
]);
if ($existingPlace) {
// Link the Place UUID to the Demande
$demande->setPlaceUuid($existingPlace->getUuidForUrl());
$demande->setPlace($existingPlace);
$place = $existingPlace;
} else {
// Create a new Place if one doesn't exist
$place = new Place();
$place->setOsmId((string)$demande->getOsmId());
$place->setOsmKind($demande->getOsmObjectType());
// Get OSM data from Overpass API
$commerce_overpass = $this->motocultrice->get_osm_object_data($demande->getOsmObjectType(), $demande->getOsmId());
if ($commerce_overpass) {
// Update the Place with OSM data
$place->update_place_from_overpass_data($commerce_overpass);
// Link the Place to the Demande
$demande->setPlaceUuid($place->getUuidForUrl());
$demande->setPlace($place);
// Persist the Place
$this->entityManager->persist($place);
}
}
// Link the Place to a Stat object using the INSEE code
if ($place && $demande->getInsee()) {
$stats = $place->getStats();
if (!$stats) {
$stats_exist = $this->entityManager->getRepository(Stats::class)->findOneBy(['zone' => $demande->getInsee()]);
if ($stats_exist) {
$stats = $stats_exist;
} else {
$stats = new Stats();
$zipcode = (string)$demande->getInsee();
$stats->setZone($zipcode);
$stats->setKind('user'); // Set the kind to 'user' as it's created from a user request
$place->setZipCode($zipcode);
$this->entityManager->persist($stats);
}
$stats->addPlace($place);
$place->setStats($stats);
}
}
}
if (!$place->getUuidForUrl()) {
$place->setUuidForUrl(uniqid());
}
if (!$place->getZipCode()) {
$place->setZipCode("00000");
}
$this->entityManager->persist($demande);
$this->entityManager->flush();
return new JsonResponse([
'success' => true,
'message' => 'Demande créée avec succès',
'id' => $demande->getId()
]);
}
#[Route('/api/demande/{id}/email', name: 'app_public_update_demande_email', methods: ['POST'])]
public function updateDemandeEmail(int $id, Request $request): JsonResponse
{
$data = json_decode($request->getContent(), true);
if (!isset($data['email']) || empty($data['email'])) {
return new JsonResponse(['success' => false, 'message' => 'L\'email est requis'], 400);
}
$demande = $this->entityManager->getRepository(Demande::class)->find($id);
if (!$demande) {
return new JsonResponse(['success' => false, 'message' => 'Demande non trouvée'], 404);
}
$demande->setEmail($data['email']);
$demande->setStatus('email_provided');
$this->entityManager->persist($demande);
$this->entityManager->flush();
return new JsonResponse([
'success' => true,
'message' => 'Email ajouté avec succès'
]);
}
#[Route('/', name: 'app_public_index')]
public function index(): Response
{
@ -129,11 +261,11 @@ class PublicController extends AbstractController
// Préparer les données pour la carte
$citiesForMap = [];
foreach ($stats as $stat) {
if ($stat->getZone() && $stat->getZone() !== 'undefined' && preg_match('/^\d+$/', $stat->getZone()) && $stat->getZone() !== '00000') {
$cityName = $stat->getName() ?: $stat->getZone();
// Utiliser les coordonnées stockées si disponibles
if ($stat->getLat() && $stat->getLon()) {
$citiesForMap[] = [
@ -167,33 +299,33 @@ class PublicController extends AbstractController
{
// Cache simple pour éviter trop d'appels API
$cacheKey = 'city_coords_' . $inseeCode;
// Vérifier le cache (ici on utilise une approche simple)
// En production, vous pourriez utiliser le cache Symfony
$query = urlencode($cityName . ', France');
$url = "https://nominatim.openstreetmap.org/search?q={$query}&format=json&limit=1&countrycodes=fr";
try {
// Ajouter un délai pour respecter les limites de l'API Nominatim
usleep(100000); // 0.1 seconde entre les appels
$context = stream_context_create([
'http' => [
'timeout' => 5, // Timeout de 5 secondes
'user_agent' => 'OSM-Commerces/1.0'
]
]);
$response = file_get_contents($url, false, $context);
if ($response === false) {
error_log("DEBUG: Échec de récupération des coordonnées pour $cityName ($inseeCode)");
return null;
}
$data = json_decode($response, true);
if (!empty($data) && isset($data[0]['lat']) && isset($data[0]['lon'])) {
error_log("DEBUG: Coordonnées trouvées pour $cityName ($inseeCode): " . $data[0]['lat'] . ", " . $data[0]['lon']);
return [
@ -206,7 +338,7 @@ class PublicController extends AbstractController
} catch (\Exception $e) {
error_log("DEBUG: Exception lors de la récupération des coordonnées pour $cityName ($inseeCode): " . $e->getMessage());
}
return null;
}
@ -269,7 +401,7 @@ class PublicController extends AbstractController
{
$this->actionLogger->log('dashboard', []);
$stats_repo = $this->entityManager->getRepository(Stats::class)->findAll();
$stats_for_chart = [];
@ -298,6 +430,75 @@ class PublicController extends AbstractController
]);
}
#[Route('/api/dashboard/regression', name: 'api_dashboard_regression', methods: ['POST'])]
public function saveRegressionData(Request $request): JsonResponse
{
$this->actionLogger->log('save_regression_data', []);
// Récupérer les données de la requête
$data = json_decode($request->getContent(), true);
if (!isset($data['angle']) || !isset($data['slope']) || !isset($data['intercept'])) {
return new JsonResponse([
'success' => false,
'message' => 'Données de régression incomplètes'
], Response::HTTP_BAD_REQUEST);
}
// Récupérer les stats globales (zone 00000)
$statsGlobal = $this->entityManager->getRepository(Stats::class)->findOneBy(['zone' => '00000']);
if (!$statsGlobal) {
// Créer les stats globales si elles n'existent pas
$statsGlobal = new Stats();
$statsGlobal->setZone('00000');
$statsGlobal->setName('toutes les villes');
$statsGlobal->setKind('request'); // Set the kind to 'request' as it's a system-generated stat
$this->entityManager->persist($statsGlobal);
$this->entityManager->flush();
}
// Créer un nouveau followup pour la régression linéaire
$followup = new CityFollowUp();
$followup->setName('regression_angle');
$followup->setMeasure($data['angle']);
$followup->setDate(new \DateTime());
$followup->setStats($statsGlobal);
$this->entityManager->persist($followup);
// Créer un followup pour la pente
$followupSlope = new CityFollowUp();
$followupSlope->setName('regression_slope');
$followupSlope->setMeasure($data['slope']);
$followupSlope->setDate(new \DateTime());
$followupSlope->setStats($statsGlobal);
$this->entityManager->persist($followupSlope);
// Créer un followup pour l'ordonnée à l'origine
$followupIntercept = new CityFollowUp();
$followupIntercept->setName('regression_intercept');
$followupIntercept->setMeasure($data['intercept']);
$followupIntercept->setDate(new \DateTime());
$followupIntercept->setStats($statsGlobal);
$this->entityManager->persist($followupIntercept);
$this->entityManager->flush();
return new JsonResponse([
'success' => true,
'message' => 'Données de régression enregistrées avec succès',
'followup' => [
'id' => $followup->getId(),
'name' => $followup->getName(),
'measure' => $followup->getMeasure(),
'date' => $followup->getDate()->format('Y-m-d H:i:s')
]
]);
}
#[Route('/modify/{osm_object_id}/{version}/{changesetID}', name: 'app_public_submit')]
public function submit($osm_object_id, $version, $changesetID): Response
{
@ -471,6 +672,7 @@ class PublicController extends AbstractController
} else {
$stats = new Stats();
$stats->setZone($place->getZipCode());
$stats->setKind('user'); // Set the kind to 'user' as it's created from a user interaction
}
}
@ -658,7 +860,7 @@ class PublicController extends AbstractController
public function publicFollowupGraph(string $insee_code, string $theme): Response
{
$stats = $this->entityManager->getRepository(Stats::class)->findOneBy(['zone' => $insee_code]);
if (!$stats) {
if (!$stats) {
$this->addFlash('error', '13 Aucune stats trouvée pour ce code INSEE.');
return $this->redirectToRoute('app_public_index');
}
@ -673,7 +875,7 @@ class PublicController extends AbstractController
$followups = $stats->getCityFollowUps();
$countData = [];
$completionData = [];
foreach ($followups as $fu) {
if ($fu->getName() === $theme . '_count') {
$countData[] = [

View file

@ -54,7 +54,7 @@ class Stats
#[ORM\Column(type: Types::INTEGER, nullable: true)]
private ?int $avec_note = null;
#[ORM\Column(length: 255, nullable: true, options: ['charset' => 'utf8mb4'] )]
#[ORM\Column(length: 255, nullable: true, options: ['charset' => 'utf8mb4'])]
private ?string $name = null;
// nombre d'habitants dans la zone
@ -118,6 +118,15 @@ class Stats
#[ORM\Column(type: 'datetime', nullable: true)]
private ?\DateTime $date_labourage_done = null;
/**
* Defines the source of the stat creation:
* - 'request': created from a request of the object Demande
* - 'command': created from a Symfony command
* - 'user': created by a user using the route to add their city
*/
#[ORM\Column(length: 20, nullable: true)]
private ?string $kind = 'command';
public function getCTCurlBase(): ?string
{
$base = 'https://complete-tes-commerces.fr/';
@ -126,7 +135,7 @@ class Stats
$departement = substr($zone, 0, 2);
$insee_code = $zone;
$slug = strtolower(str_replace(' ', '-', $this->getName()));
$url = $base . $departement . '/' . $insee_code . '-'.$slug.'/json/' . $slug ;
$url = $base . $departement . '/' . $insee_code . '-' . $slug . '/json/' . $slug;
return $url;
}
@ -140,7 +149,7 @@ class Stats
{
return $this->getParametricJsonFromCTC('_dailystats');
}
public function getOSMClosedSirets(): string
{
return $this->getParametricJsonFromCTC('_osm_closed_siret');
@ -171,15 +180,14 @@ class Stats
return $this->getParametricJsonFromCTC('_sirene_matches');
}
public function getParametricJsonFromCTC($suffixe): string
{
$url = $this->getCTCurlBase().$suffixe.'.json';
$url = $this->getCTCurlBase() . $suffixe . '.json';
return $url;
}
public function getAllCTCUrlsMap(): ?array
{
@ -197,7 +205,7 @@ class Stats
}
return $urls;
}
// calcule le pourcentage de complétion de la zone
public function computeCompletionPercent(): ?int
@ -218,9 +226,9 @@ class Stats
$this->avec_note = 0;
$this->avec_siret = 0;
$this->avec_name = 0;
$somme_completions = 0;
// On boucle sur chaque place pour compter les attributs renseignés
foreach ($this->places as $place) {
@ -230,7 +238,7 @@ class Stats
$place_completions++;
}
if ($place->hasWebsite()) {
$this->avec_site++;
$this->avec_site++;
$place_completions++;
}
if ($place->hasWheelchair()) {
@ -241,19 +249,19 @@ class Stats
$this->avec_horaires++;
$place_completions++;
}
if($place->getSiret()) {
if ($place->getSiret()) {
$this->avec_siret++;
$place_completions++;
}
if($place->getName()) {
if ($place->getName()) {
$this->avec_name++;
$place_completions++;
}
if($place->getNoteContent()) {
if ($place->getNoteContent()) {
$this->avec_note++;
// on ne compte pas les notes comme indice de complétion
}
$somme_completions += $place_completions / 6;
}
@ -263,9 +271,9 @@ class Stats
}
public function __construct()
{
$this->setKind('command');
$this->places = new ArrayCollection();
$this->statsHistories = new ArrayCollection();
$this->cityFollowUps = new ArrayCollection();
@ -365,7 +373,7 @@ class Stats
return $this;
}
public function getAvecSite(): ?int
{
return $this->avec_site;
@ -377,7 +385,7 @@ class Stats
return $this;
}
public function getAvecAccessibilite(): ?int
{
return $this->avec_accessibilite;
@ -635,24 +643,37 @@ class Stats
$this->lon = $lon;
return $this;
}
public function getDateLabourageRequested(): ?\DateTime
{
return $this->date_labourage_requested;
}
public function setDateLabourageRequested(?\DateTime $date): static
{
$this->date_labourage_requested = $date;
return $this;
}
public function getDateLabourageDone(): ?\DateTime
{
return $this->date_labourage_done;
}
public function setDateLabourageDone(?\DateTime $date): static
{
$this->date_labourage_done = $date;
return $this;
}
}
public function getKind(): ?string
{
return $this->kind;
}
public function setKind(?string $kind): static
{
$this->kind = $kind;
return $this;
}
}