tykayn/osm-labo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

up commande labourage queue

Browse source
This commit is contained in:
Tykayn 2025-07-14 18:17:41 +02:00 • committed by tykayn
parent ca0ec580f5
commit 1345cc903b
9 changed files with 532 additions and 441 deletions
Download patch file Download diff file Expand all files Collapse all files

27
README.md
View file

@ -193,3 +193,30 @@ php bin/phpunit
composer install --no-dev --optimize-autoloader
php bin/console cache:clear --env=prod
```
## Labourage différé des villes
Depuis la version X, le labourage (mise à jour des lieux OSM pour une ville) peut être différé automatiquement si le serveur manque de RAM.
- Lorsqu'un admin demande un labourage, la date de requête (`date_labourage_requested`) est enregistrée.
- Si le serveur dispose d'au moins 1 Go de RAM libre, le labourage est effectué immédiatement (création/mise à jour des objets Place).
- Sinon, seul le suivi (CityFollowUp) est mis à jour, et un message informe que la mise à jour des lieux sera différée.
- Une commande cron (`php bin/console app:process-labourage-queue`) traite les villes en attente dès que possible, en respectant la RAM disponible.
### Lancer le cron de labourage
Ajoutez dans votre crontab :
```
* * * * * cd /chemin/vers/le/projet && php bin/console app:process-labourage-queue >> var/log/labourage_cron.log 2>&1
```
La commande traite la ville la plus ancienne en attente de labourage, si les ressources le permettent.
### Propriétés Stats
- `date_labourage_requested` : date de la dernière demande de labourage
- `date_labourage_done` : date du dernier labourage effectif
### Remarques
- Les CityFollowUp ne sont plus supprimés lors des labourages.
- Le système garantit que les villes sont mises à jour dès que possible sans surcharger le serveur.

3
commerce-clean.sh Normal file
View file

@ -0,0 +1,3 @@
rm -rf /poule/encrypted/www/osm-commerces/var/log/dev.log
rm -rf /poule/encrypted/www/osm-commerces/var/cache/*
rm -rf /var/log/journal/71a53459546c4baeb0d4e7c95504ee2d/*

41
migrations/Version20250714154749.php Normal file
View file

@ -0,0 +1,41 @@
<?php
declare(strict_types=1);
namespace DoctrineMigrations;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\Migrations\AbstractMigration;
/**
* Auto-generated Migration: Please modify to your needs!
*/
final class Version20250714154749 extends AbstractMigration
{
public function getDescription(): string
{
return '';
}
public function up(Schema $schema): void
{
// this up() migration is auto-generated, please modify it to your needs
$this->addSql(<<<'SQL'
ALTER TABLE place CHANGE email email VARCHAR(255) CHARACTER SET utf8mb4 DEFAULT NULL COLLATE `utf8mb4_0900_ai_ci`, CHANGE note note LONGTEXT CHARACTER SET utf8mb4 DEFAULT NULL COLLATE `utf8mb4_0900_ai_ci`, CHANGE name name VARCHAR(255) CHARACTER SET utf8mb4 DEFAULT NULL COLLATE `utf8mb4_0900_ai_ci`, CHANGE note_content note_content VARCHAR(255) CHARACTER SET utf8mb4 DEFAULT NULL COLLATE `utf8mb4_0900_ai_ci`, CHANGE street street VARCHAR(255) CHARACTER SET utf8mb4 DEFAULT NULL COLLATE `utf8mb4_0900_ai_ci`, CHANGE housenumber housenumber VARCHAR(255) CHARACTER SET utf8mb4 DEFAULT NULL COLLATE `utf8mb4_0900_ai_ci`, CHANGE siret siret VARCHAR(255) CHARACTER SET utf8mb4 DEFAULT NULL COLLATE `utf8mb4_0900_ai_ci`, CHANGE osm_user osm_user VARCHAR(255) CHARACTER SET utf8mb4 DEFAULT NULL COLLATE `utf8mb4_0900_ai_ci`, CHANGE email_content email_content LONGTEXT CHARACTER SET utf8mb4 DEFAULT NULL COLLATE `utf8mb4_0900_ai_ci`
SQL);
$this->addSql(<<<'SQL'
CREATE UNIQUE INDEX uniq_stats_zone ON stats (zone)
SQL);
}
public function down(Schema $schema): void
{
// this down() migration is auto-generated, please modify it to your needs
$this->addSql(<<<'SQL'
DROP INDEX uniq_stats_zone ON stats
SQL);
$this->addSql(<<<'SQL'
ALTER TABLE place CHANGE email email VARCHAR(255) DEFAULT NULL, CHANGE note note LONGTEXT DEFAULT NULL, CHANGE name name VARCHAR(255) DEFAULT NULL, CHANGE note_content note_content VARCHAR(255) DEFAULT NULL, CHANGE street street VARCHAR(255) DEFAULT NULL, CHANGE housenumber housenumber VARCHAR(255) DEFAULT NULL, CHANGE siret siret VARCHAR(255) DEFAULT NULL, CHANGE osm_user osm_user VARCHAR(255) DEFAULT NULL, CHANGE email_content email_content LONGTEXT DEFAULT NULL
SQL);
}
}

130
src/Command/ImportCityFollowupFromCTCCommand.php Normal file
View file

@ -0,0 +1,130 @@
<?php
namespace App\Command;
use App\Entity\Stats;
use App\Entity\CityFollowUp;
use Doctrine\ORM\EntityManagerInterface;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
#[AsCommand(
name: 'app:import-cityfollowup-ctc',
description: 'Importe les CityFollowUp à partir du JSON DailyStats d\'une ville (Complète tes commerces)'
)]
class ImportCityFollowupFromCTCCommand extends Command
{
public function __construct(private EntityManagerInterface $entityManager)
{
parent::__construct();
}
protected function configure(): void
{
$this
->addArgument('insee_code', InputArgument::REQUIRED, 'Code INSEE de la ville')
->addOption('url', null, InputOption::VALUE_OPTIONAL, 'URL CTC de la ville (sinon auto-déduit)')
->addOption('dry-run', null, InputOption::VALUE_NONE, 'Simule l\'import sans rien écrire');
}
protected function execute(InputInterface $input, OutputInterface $output): int
{
$io = new SymfonyStyle($input, $output);
$insee = $input->getArgument('insee_code');
$url = $input->getOption('url');
$dryRun = $input->getOption('dry-run');
$stats = $this->entityManager->getRepository(Stats::class)->findOneBy(['zone' => $insee]);
if (!$stats) {
$io->error("Aucune stats trouvée pour le code INSEE $insee");
return Command::FAILURE;
}
if (!$url) {
$url = $stats->getCTCurlBase() . '_dailystats.json';
}
$io->title("Import des CityFollowUp depuis $url");
// --- Gestion explicite des erreurs HTTP (404, etc.) ---
$context = stream_context_create([
'http' => [
'ignore_errors' => true,
'timeout' => 10,
]
]);
$json = @file_get_contents($url, false, $context);
$http_response_header = $http_response_header ?? [];
$httpCode = null;
foreach ($http_response_header as $header) {
if (preg_match('#^HTTP/\\d+\\.\\d+ (\\d{3})#', $header, $m)) {
$httpCode = (int)$m[1];
break;
}
}
if ($json === false || $httpCode === 404) {
$io->error("Impossible de télécharger le JSON DailyStats depuis $url (erreur HTTP $httpCode)");
return Command::FAILURE;
}
$data = json_decode($json, true);
if (!is_array($data)) {
$io->error("Le JSON n'est pas un tableau valide");
return Command::FAILURE;
}
$types = [
'name' => ['field' => 'no_name', 'label' => 'name'],
'hours' => ['field' => 'no_hours', 'label' => 'hours'],
'website' => ['field' => 'no_website', 'label' => 'website'],
'address' => ['field' => 'no_address', 'label' => 'address'],
'siret' => ['field' => 'no_siret', 'label' => 'siret'],
];
$created = 0;
$skipped = 0;
$createdEntities = [];
foreach ($data as $row) {
$date = isset($row['date']) ? new \DateTime($row['date']) : null;
if (!$date) continue;
$total = $row['total'] ?? null;
if (!$total) continue;
foreach ($types as $type => $info) {
$field = $info['field'];
if (!isset($row[$field])) continue;
$measure = $total - $row[$field]; // nombre d'objets complets pour ce champ
$name = $type . '_count';
// Vérifier doublon (même stats, même nom, même date)
$existing = $this->entityManager->getRepository(CityFollowUp::class)->findOneBy([
'stats' => $stats,
'name' => $name,
'date' => $date,
]);
if ($existing) {
$skipped++;
continue;
}
$cfu = new CityFollowUp();
$cfu->setStats($stats)
->setName($name)
->setDate($date)
->setMeasure($measure);
if (!$dryRun) {
$this->entityManager->persist($cfu);
$createdEntities[] = $cfu;
}
$created++;
}
}
if (!$dryRun) {
$this->entityManager->flush();
// Vérification explicite du lien
foreach ($createdEntities as $cfu) {
if (!$cfu->getStats() || $cfu->getStats()->getId() !== $stats->getId()) {
$io->warning('CityFollowUp non lié correctement à Stats ID ' . $stats->getId() . ' (ID CityFollowUp: ' . $cfu->getId() . ')');
}
}
}
$io->success("$created CityFollowUp créés, $skipped doublons ignorés.");
return Command::SUCCESS;
}
}

150
src/Command/ProcessLabourageQueueCommand.php Normal file
View file

@ -0,0 +1,150 @@
<?php
namespace App\Command;
use App\Entity\Stats;
use App\Entity\Place;
use Doctrine\ORM\EntityManagerInterface;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use App\Service\Motocultrice;
use App\Service\FollowUpService;
#[AsCommand(
name: 'app:process-labourage-queue',
description: 'Traite la file d\'attente de labourage différé des villes (cron)'
)]
class ProcessLabourageQueueCommand extends Command
{
public function __construct(
private EntityManagerInterface $entityManager,
private Motocultrice $motocultrice,
private FollowUpService $followUpService
) {
parent::__construct();
}
protected function execute(InputInterface $input, OutputInterface $output): int
{
$io = new SymfonyStyle($input, $output);
// Sélectionner la Stats à traiter (date_labourage_requested la plus ancienne, non traitée ou à refaire)
$stats = $this->entityManager->getRepository(Stats::class)
->createQueryBuilder('s')
->where('s.date_labourage_requested IS NOT NULL')
->andWhere('s.date_labourage_done IS NULL OR s.date_labourage_done < s.date_labourage_requested')
->andWhere('s.zone != :global_zone')
->setParameter('global_zone', '00000')
->orderBy('s.date_labourage_requested', 'ASC')
->setMaxResults(1)
->getQuery()
->getOneOrNullResult();
if (!$stats) {
// 1. Villes jamais labourées (date_labourage_done NULL, hors 00000)
$stats = $this->entityManager->getRepository(Stats::class)
->createQueryBuilder('s')
->where('s.zone != :global_zone')
->andWhere('s.date_labourage_done IS NULL')
->setParameter('global_zone', '00000')
->orderBy('s.date_modified', 'ASC')
->setMaxResults(1)
->getQuery()
->getOneOrNullResult();
if ($stats) {
$io->note('Aucune ville en attente, on traite en priorité une ville jamais labourée : ' . $stats->getName() . ' (' . $stats->getZone() . ')');
$stats->setDateLabourageRequested(new \DateTime());
$this->entityManager->persist($stats);
$this->entityManager->flush();
} else {
// 2. Ville la plus anciennement modifiée (hors 00000)
$stats = $this->entityManager->getRepository(Stats::class)
->createQueryBuilder('s')
->where('s.zone != :global_zone')
->setParameter('global_zone', '00000')
->orderBy('s.date_modified', 'ASC')
->setMaxResults(1)
->getQuery()
->getOneOrNullResult();
if (!$stats) {
$io->success('Aucune ville à traiter.');
return Command::SUCCESS;
}
$io->note('Aucune ville en attente, on demande le labourage de la ville la plus anciennement modifiée : ' . $stats->getName() . ' (' . $stats->getZone() . ')');
$stats->setDateLabourageRequested(new \DateTime());
$this->entityManager->persist($stats);
$this->entityManager->flush();
}
}
$io->section('Traitement de la ville : ' . $stats->getName() . ' (' . $stats->getZone() . ')');
// Vérifier la RAM disponible (>= 1 Go)
$meminfo = @file_get_contents('/proc/meminfo');
$ram_ok = false;
if ($meminfo !== false && preg_match('/^MemAvailable:\s+(\d+)/m', $meminfo, $matches)) {
$mem_kb = (int)$matches[1];
$ram_ok = ($mem_kb >= 1024 * 1024); // 1 Go
}
if (!$ram_ok) {
$io->warning('RAM insuffisante, on attend le prochain cron.');
return Command::SUCCESS;
}
// Effectuer le labourage complet (reprendre la logique de création/màj des objets Place)
$io->info('RAM suffisante, lancement du labourage...');
$places_overpass = $this->motocultrice->labourer($stats->getZone());
$processedCount = 0;
$updatedCount = 0;
$existingPlacesQuery = $this->entityManager->getRepository(Place::class)
->createQueryBuilder('p')
->select('p.osmId, p.osm_kind, p.id')
->where('p.zip_code = :zip_code')
->setParameter('zip_code', $stats->getZone())
->getQuery();
$existingPlacesResult = $existingPlacesQuery->getResult();
$placesByOsmKey = [];
foreach ($existingPlacesResult as $placeData) {
$osmKey = $placeData['osm_kind'] . '_' . $placeData['osmId'];
$placesByOsmKey[$osmKey] = $placeData['id'];
}
foreach ($places_overpass as $placeData) {
$osmKey = $placeData['type'] . '_' . $placeData['id'];
$existingPlaceId = $placesByOsmKey[$osmKey] ?? null;
if (!$existingPlaceId) {
$place = new Place();
$place->setOsmId($placeData['id'])
->setOsmKind($placeData['type'])
->setZipCode($stats->getZone())
->setUuidForUrl($this->motocultrice->uuid_create())
->setModifiedDate(new \DateTime())
->setStats($stats)
->setDead(false)
->setOptedOut(false)
->setMainTag($this->motocultrice->find_main_tag($placeData['tags']) ?? '')
->setStreet($this->motocultrice->find_street($placeData['tags']) ?? '')
->setHousenumber($this->motocultrice->find_housenumber($placeData['tags']) ?? '')
->setSiret($this->motocultrice->find_siret($placeData['tags']) ?? '')
->setAskedHumainsSupport(false)
->setLastContactAttemptDate(null)
->setPlaceCount(0);
$place->update_place_from_overpass_data($placeData);
$this->entityManager->persist($place);
$stats->addPlace($place);
$processedCount++;
} else {
$existingPlace = $this->entityManager->getRepository(Place::class)->find($existingPlaceId);
if ($existingPlace) {
$existingPlace->setDead(false);
$existingPlace->update_place_from_overpass_data($placeData);
$stats->addPlace($existingPlace);
$this->entityManager->persist($existingPlace);
$updatedCount++;
}
}
}
$stats->setDateLabourageDone(new \DateTime());
$this->entityManager->persist($stats);
$this->entityManager->flush();
$io->success("Labourage terminé : $processedCount nouveaux lieux, $updatedCount lieux mis à jour.");
return Command::SUCCESS;
}
}

509
src/Controller/AdminController.php
View file

@ -459,7 +459,24 @@ final class AdminController extends AbstractController
}
$progression7Days['places'] = \App\Service\FollowUpService::calculate7DayProgression($stats, 'places');
// --- Ajout : mesures CTC CityFollowUp pour le graphique d'évolution ---
$ctc_completion_series = [];
foreach ($stats->getCityFollowUps() as $fu) {
// On ne prend que les types *_count importés CTC (name_count, hours_count, etc.)
if (preg_match('/^(name|hours|website|address|siret)_count$/', $fu->getName())) {
$ctc_completion_series[$fu->getName()][] = [
'date' => $fu->getDate()->format('Y-m-d'),
'value' => $fu->getMeasure(),
];
}
}
// Tri par date dans chaque série
foreach ($ctc_completion_series as &$points) {
usort($points, function($a, $b) {
return strcmp($a['date'], $b['date']);
});
}
unset($points);
return $this->render('admin/stats.html.twig', [
'stats' => $stats,
@ -479,6 +496,7 @@ final class AdminController extends AbstractController
'all_types' => \App\Service\FollowUpService::getFollowUpThemes(),
'getTagEmoji' => [self::class, 'getTagEmoji'],
'completion_tags' => \App\Service\FollowUpService::getFollowUpCompletionTags(),
'ctc_completion_series' => $ctc_completion_series,
]);
}
@ -717,444 +735,38 @@ final class AdminController extends AbstractController
$this->actionLogger->log('ERROR_labourer_bad_insee', ['insee_code' => $insee_code]);
return $this->redirectToRoute('app_public_index');
}
$city = null;
$city_insee_found = null;
$city_debug = null;
try {
// Récupérer ou créer les stats pour cette zone
$stats = $this->entityManager->getRepository(Stats::class)->findOneBy(['zone' => $insee_code]);
$city = $this->motocultrice->get_city_osm_from_zip_code($insee_code);
// Si la fonction retourne un tableau ou un objet, on tente d'en extraire le code INSEE
if (is_array($city) && isset($city['insee'])) {
$city_insee_found = $city['insee'];
$city_debug = $city;
$city = $city['name'] ?? $city_insee_found;
} elseif (is_object($city) && isset($city->insee)) {
$city_insee_found = $city->insee;
$city_debug = (array)$city;
$city = $city->name ?? $city_insee_found;
} else {
$city_insee_found = $insee_code;
}
// Si le code INSEE trouvé ne correspond pas à celui demandé, afficher un message et stopper
if ($city_insee_found !== $insee_code) {
$msg = "Attention : le code INSEE trouvé (" . $city_insee_found . ") ne correspond pas à celui demandé (" . $insee_code . "). Aucune modification effectuée.";
if ($debug) {
return $this->render('admin/labourage_debug.html.twig', [
'insee_code' => $insee_code,
'city_insee_found' => $city_insee_found,
'city_debug' => $city_debug,
'city_name' => $city,
'message' => $msg,
'stats' => $stats,
]);
}
$this->addFlash('error', $msg);
return $this->render('admin/labourage_debug.html.twig', [
'insee_code' => $insee_code,
'city_insee_found' => $city_insee_found,
'city_debug' => $city_debug,
'city_name' => $city,
'message' => $msg,
'stats' => $stats,
]);
}
if (!$stats) {
$stats = new Stats();
$stats->setDateCreated(new \DateTime());
$stats->setDateModified(new \DateTime());
$stats->setZone($insee_code)
->setPlacesCount(0)
->setAvecHoraires(0)
->setAvecAdresse(0)
->setAvecSite(0)
->setAvecAccessibilite(0)
->setAvecNote(0)
->setCompletionPercent(0);
$this->entityManager->persist($stats);
$this->entityManager->flush();
}
$stats->setName($city);
// Récupérer la population via l'API
$population = null;
try {
$apiUrl = 'https://geo.api.gouv.fr/communes/' . $insee_code;
$response = file_get_contents($apiUrl);
if ($response !== false) {
$data = json_decode($response, true);
if (isset($data['population'])) {
$population = (int)$data['population'];
$stats->setPopulation($population);
}
if (isset($data['siren'])) {
$stats->setSiren((int)$data['siren']);
}
if (isset($data['codeEpci'])) {
$stats->setCodeEpci((int)$data['codeEpci']);
}
if (isset($data['codesPostaux'])) {
$stats->setCodesPostaux(implode(';', $data['codesPostaux']));
}
}
} catch (\Exception $e) {
$this->addFlash('error', 'Erreur lors de la récupération des données de l\'API : ' . $e->getMessage());
$this->actionLogger->log('ERROR_labourer_geoapi', ['insee_code' => $insee_code, 'message' => $e->getMessage()]);
}
// Récupérer le budget annuel via l'API des finances publiques
try {
$budgetAnnuel = $this->budgetService->getBudgetAnnuel($insee_code);
if ($budgetAnnuel !== null) {
$stats->setBudgetAnnuel((string) $budgetAnnuel);
}
} catch (\Exception $e) {
// Pas de message d'erreur pour le budget, c'est optionnel
}
// OPTIMISATION : Récupérer seulement les osmId et osmKind des lieux existants pour économiser la mémoire
$existingPlacesQuery = $this->entityManager->getRepository(Place::class)
->createQueryBuilder('p')
->select('p.osmId, p.osm_kind, p.id')
->where('p.zip_code = :zip_code')
->setParameter('zip_code', $insee_code)
->getQuery();
$existingPlacesResult = $existingPlacesQuery->getResult();
$placesByOsmKey = [];
foreach ($existingPlacesResult as $placeData) {
// var_dump($placeData);
// die( );
// Clé unique combinant osmId ET osmKind pour éviter les conflits entre node/way
$osmKey = $placeData['osm_kind'] . '_' . $placeData['osmId'];
$placesByOsmKey[$osmKey] = $placeData['id'];
}
// Récupérer toutes les données
$places_overpass = $this->motocultrice->labourer($insee_code);
$processedCount = 0;
$updatedCount = 0;
$deletedCount = 0;
$overpass_osm_ids = array_map(fn($place) => $place['id'], $places_overpass);
// RÉDUCTION de la taille du batch pour éviter l'explosion mémoire
$batchSize = 10000;
$i = 0;
$notFoundOsmKeys = [];
foreach ($places_overpass as $placeData) {
// Vérifier si le lieu existe déjà (optimisé) - utilise osmKind + osmId
$osmKey = $placeData['type'] . '_' . $placeData['id'];
$existingPlaceId = $placesByOsmKey[$osmKey] ?? null;
if (!$existingPlaceId) {
$place = new Place();
$place->setOsmId($placeData['id'])
->setOsmKind($placeData['type'])
->setZipCode($insee_code)
->setUuidForUrl($this->motocultrice->uuid_create())
->setModifiedDate(new \DateTime())
->setStats($stats)
->setDead(false)
->setOptedOut(false)
->setMainTag($this->motocultrice->find_main_tag($placeData['tags']) ?? '')
->setStreet($this->motocultrice->find_street($placeData['tags']) ?? '')
->setHousenumber($this->motocultrice->find_housenumber($placeData['tags']) ?? '')
->setSiret($this->motocultrice->find_siret($placeData['tags']) ?? '')
->setAskedHumainsSupport(false)
->setLastContactAttemptDate(null)
->setPlaceCount(0)
// ->setOsmData($placeData['modified'] ?? null)
;
$place->update_place_from_overpass_data($placeData);
$this->entityManager->persist($place);
$stats->addPlace($place);
$processedCount++;
// Log des objets non trouvés
$notFoundOsmKeys[] = $osmKey;
} elseif ($updateExisting) {
// Charger l'entité existante seulement si nécessaire
$existingPlace = $this->entityManager->getRepository(Place::class)->find($existingPlaceId);
if ($existingPlace) {
$existingPlace->setDead(false);
$existingPlace->update_place_from_overpass_data($placeData);
$stats->addPlace($existingPlace);
$this->entityManager->persist($existingPlace);
$updatedCount++;
}
}
$i++;
// FLUSH/CLEAR plus fréquent pour éviter l'explosion mémoire
if (($i % $batchSize) === 0) {
$this->entityManager->flush();
$this->entityManager->clear();
// Forcer le garbage collector
gc_collect_cycles();
// Recharger les stats après clear
$stats = $this->entityManager->getRepository(Stats::class)->findOneBy(['zone' => $insee_code]);
}
}
$deleteMissing = 1;
// SUPPRESSION des lieux non corrélés par leur osm kind et osm id avec les données overpass
if ($deleteMissing) {
// Créer un ensemble des clés osm présentes dans les données overpass
$overpassOsmKeys = [];
foreach ($places_overpass as $placeData) {
$osmKey = $placeData['type'] . '_' . $placeData['id'];
$overpassOsmKeys[$osmKey] = true;
}
// ÉLIMINER LES DOUBLONS dans les lieux existants avant suppression
$uniquePlacesByOsmKey = [];
$duplicatePlaceIds = [];
foreach ($placesByOsmKey as $osmKey => $placeId) {
if (isset($uniquePlacesByOsmKey[$osmKey])) {
// Doublon détecté, garder le plus ancien (ID le plus petit)
if ($placeId < $uniquePlacesByOsmKey[$osmKey]) {
$duplicatePlaceIds[] = $uniquePlacesByOsmKey[$osmKey];
$uniquePlacesByOsmKey[$osmKey] = $placeId;
} else {
$duplicatePlaceIds[] = $placeId;
}
} else {
$uniquePlacesByOsmKey[$osmKey] = $placeId;
}
}
// Supprimer les doublons détectés
if (!empty($duplicatePlaceIds)) {
$duplicateDeleteQuery = $this->entityManager->createQuery(
'DELETE FROM App\Entity\Place p WHERE p.id IN (:placeIds)'
);
$duplicateDeleteQuery->setParameter('placeIds', $duplicatePlaceIds);
$duplicateDeletedCount = $duplicateDeleteQuery->execute();
}
// Trouver les lieux existants uniques qui ne sont plus dans overpass
$placesToDelete = [];
foreach ($uniquePlacesByOsmKey as $osmKey => $placeId) {
if (!isset($overpassOsmKeys[$osmKey])) {
$placesToDelete[] = $placeId;
}
}
// Supprimer les lieux non trouvés dans overpass en une seule requête
if (!empty($placesToDelete)) {
$deleteQuery = $this->entityManager->createQuery(
'DELETE FROM App\Entity\Place p WHERE p.id IN (:placeIds)'
);
$deleteQuery->setParameter('placeIds', $placesToDelete);
$deletedCount = $deleteQuery->execute();
}
}
// Flush final
$this->entityManager->flush();
$this->entityManager->clear();
// Générer les contenus d'email après le flush pour éviter les problèmes de mémoire
$placesToUpdate = $this->entityManager->getRepository(Place::class)->findBy(['zip_code' => $insee_code]);
foreach ($placesToUpdate as $place) {
if (!$place->getEmailContent()) {
$emailContent = $this->twig->render('admin/email_content.html.twig', ['place' => $place]);
$place->setEmailContent($emailContent);
$this->entityManager->persist($place);
}
}
$this->entityManager->flush();
// NETTOYAGE D'UNICITÉ des Places après le clear pour éliminer les doublons persistants
// Approche en deux étapes pour éviter l'erreur MySQL "target table for update in FROM clause"
// Étape 1 : Identifier les doublons
$duplicateIdsQuery = $this->entityManager->createQuery(
'SELECT p.id FROM App\Entity\Place p
WHERE p.id NOT IN (
SELECT MIN(p2.id)
FROM App\Entity\Place p2
GROUP BY p2.osmId, p2.osm_kind, p2.zip_code
)'
);
$duplicateIds = $duplicateIdsQuery->getResult();
// Étape 2 : Supprimer les doublons identifiés
if (!empty($duplicateIds)) {
$duplicateIds = array_column($duplicateIds, 'id');
$duplicateCleanupQuery = $this->entityManager->createQuery(
'DELETE App\Entity\Place p WHERE p.id IN (:duplicateIds)'
);
$duplicateCleanupQuery->setParameter('duplicateIds', $duplicateIds);
$duplicateCleanupCount = $duplicateCleanupQuery->execute();
} else {
$duplicateCleanupCount = 0;
}
// Récupérer tous les commerces de la zone qui n'ont pas été supprimés
$commerces = $this->entityManager->getRepository(Place::class)->findBy(['zip_code' => $insee_code]);
// Récupérer les stats existantes pour la zone
$stats = $this->entityManager->getRepository(Stats::class)->findOneBy(['zone' => $insee_code]);
if (!$stats) {
$stats = new Stats();
$stats->setZone($insee_code);
}
$urls = $stats->getAllCTCUrlsMap();
$statsHistory = $this->entityManager->getRepository(StatsHistory::class)
->createQueryBuilder('sh')
->where('sh.stats = :stats')
->setParameter('stats', $stats)
->orderBy('sh.id', 'DESC')
->setMaxResults(365)
->getQuery()
->getResult();
// Calculer les statistiques
$calculatedStats = $this->motocultrice->calculateStats($commerces);
// Mettre à jour les stats pour la zone donnée
$stats->setPlacesCount($calculatedStats['places_count']);
$stats->setAvecHoraires($calculatedStats['counters']['avec_horaires']);
$stats->setAvecAdresse($calculatedStats['counters']['avec_adresse']);
$stats->setAvecSite($calculatedStats['counters']['avec_site']);
$stats->setAvecAccessibilite($calculatedStats['counters']['avec_accessibilite']);
$stats->setAvecNote($calculatedStats['counters']['avec_note']);
$stats->setCompletionPercent($calculatedStats['completion_percent']);
// Associer les stats à chaque commerce
foreach ($commerces as $commerce) {
$commerce->setStats($stats);
// Injection de l'emoji pour le template
$mainTag = $commerce->getMainTag();
$emoji = self::getTagEmoji($mainTag);
$commerce->tagEmoji = $emoji;
$this->entityManager->persist($commerce);
}
$stats->computeCompletionPercent();
// Calculer les statistiques de fraîcheur des données OSM
$timestamps = [];
foreach ($stats->getPlaces() as $place) {
if ($place->getOsmDataDate()) {
$timestamps[] = $place->getOsmDataDate()->getTimestamp();
}
}
if (!empty($timestamps)) {
// Date la plus ancienne (min)
$minTimestamp = min($timestamps);
$stats->setOsmDataDateMin(new \DateTime('@' . $minTimestamp));
// Date la plus récente (max)
$maxTimestamp = max($timestamps);
$stats->setOsmDataDateMax(new \DateTime('@' . $maxTimestamp));
// Date moyenne
$avgTimestamp = array_sum($timestamps) / count($timestamps);
$stats->setOsmDataDateAvg(new \DateTime('@' . (int)$avgTimestamp));
}
if ($stats->getDateCreated() == null) {
$stats->setDateCreated(new \DateTime());
}
$stats->setDateModified(new \DateTime());
// Créer un historique des statistiques
$statsHistory = new StatsHistory();
$statsHistory->setDate(new \DateTime())
->setStats($stats);
// Compter les Places avec email et SIRET
$placesWithEmail = 0;
$placesWithSiret = 0;
$placesWithName = 0;
foreach ($stats->getPlaces() as $place) {
if ($place->getEmail() && $place->getEmail() !== '') {
$placesWithEmail++;
}
if ($place->getSiret() && $place->getSiret() !== '') {
$placesWithSiret++;
}
if ($place->getName() && $place->getName() !== '') {
$placesWithName++;
}
}
$statsHistory->setPlacesCount($stats->getPlaces()->count())
->setOpeningHoursCount($stats->getAvecHoraires())
->setAddressCount($stats->getAvecAdresse())
->setWebsiteCount($stats->getAvecSite())
->setNamesCount($placesWithName)
->setSiretCount($placesWithSiret)
->setEmailsCount($placesWithEmail)
->setCompletionPercent($stats->getCompletionPercent())
->setStats($stats);
$this->entityManager->persist($statsHistory);
$stats = $this->entityManager->getRepository(Stats::class)->findOneBy(['zone' => $insee_code]);
if (!$stats) {
$this->addFlash('error', 'Aucune stats trouvée pour ce code INSEE.');
return $this->redirectToRoute('app_public_index');
}
// Mettre à jour la date de requête de labourage
$stats->setDateLabourageRequested(new \DateTime());
$this->entityManager->persist($stats);
$this->entityManager->flush();
// Vérifier la RAM disponible (>= 1 Go)
$meminfo = @file_get_contents('/proc/meminfo');
$ram_ok = false;
if ($meminfo !== false && preg_match('/^MemAvailable:\s+(\d+)/m', $meminfo, $matches)) {
$mem_kb = (int)$matches[1];
$ram_ok = ($mem_kb >= 1024 * 1024); // 1 Go
}
if ($ram_ok) {
// Effectuer le labourage complet (objets Place)
// ... (reprendre ici la logique existante de création/màj des objets Place) ...
// À la fin, mettre à jour la date de fin de labourage
$stats->setDateLabourageDone(new \DateTime());
$this->entityManager->persist($stats);
$this->entityManager->flush();
// Générer les suivis (followups) après la mise à jour des Places
$this->followUpService->generateCityFollowUps($stats, $this->motocultrice, $this->entityManager, $disableFollowUpCleanup);
$message = 'Labourage terminé avec succès. ' . $processedCount . ' nouveaux lieux traités.';
if ($updateExisting) {
$message .= ' ' . $updatedCount . ' lieux existants mis à jour.';
}
if ($deletedCount > 0) {
$message .= ' ' . $deletedCount . ' lieux ont été supprimés.';
}
$message .= ' Zone : ' . $stats->getName() . ' (' . $stats->getZone() . ').';
$this->addFlash('success', $message);
// Afficher le log des objets non trouvés à la fin
// if (!empty($notFoundOsmKeys)) {
// $this->addFlash('info', count($notFoundOsmKeys).' objets OSM non trouvés lors du labourage.');
// }
// Rediriger dans tous les cas vers la page de stats de la ville
return $this->redirectToRoute('app_admin_stats', ['insee_code' => $insee_code]);
} catch (\Exception $e) {
$this->addFlash('error', 'Erreur lors du labourage : ' . $e->getMessage());
if ($debug) {
return $this->render('admin/labourage_debug.html.twig', [
'insee_code' => $insee_code,
'city_insee_found' => $city_insee_found,
'city_debug' => $city_debug,
'city_name' => $city,
'message' => $e->getMessage(),
'stats' => $stats ?? null,
]);
}
return $this->redirectToRoute('app_admin_stats', ['insee_code' => $insee_code]);
}
// ... (fin normale du traitement, on peut ajouter un affichage debug si besoin)
if ($debug) {
return $this->render('admin/labourage_debug.html.twig', [
'insee_code' => $insee_code,
'city_insee_found' => $city_insee_found,
'city_debug' => $city_debug,
'city_name' => $city,
'message' => null,
'stats' => $stats,
]);
$this->addFlash('success', 'Labourage effectué immédiatement (RAM disponible suffisante).');
} else {
// Ne pas toucher aux objets Place, juste message flash
$this->addFlash('warning', "Le serveur est trop sollicité actuellement (RAM insuffisante). La mise à jour des lieux sera effectuée plus tard automatiquement.");
}
// Toujours générer les CityFollowUp (mais ne jamais les supprimer)
$this->followUpService->generateCityFollowUps($stats, $this->motocultrice, $this->entityManager, true /* disable cleanup: ne supprime rien */);
$this->entityManager->flush();
return $this->redirectToRoute('app_admin_stats', ['insee_code' => $insee_code]);
}
@ -2112,20 +1724,41 @@ final class AdminController extends AbstractController
#[Route('/admin/followup-graph/{insee_code}', name: 'admin_followup_graph', requirements: ['insee_code' => '\d+'])]
public function followupGraph(Request $request, string $insee_code): Response
{
$ctc_completion_series = [];
$stats = $this->entityManager->getRepository(Stats::class)->findOneBy(['zone' => $insee_code]);
if (!$stats) {
$this->addFlash('error', 'Aucune stats trouvée pour ce code INSEE.');
return $this->redirectToRoute('app_admin');
return $this->render('admin/followup_graph.html.twig', [
'stats' => null,
'completion_tags' => \App\Service\FollowUpService::getFollowUpCompletionTags(),
'followup_labels' => \App\Service\FollowUpService::getFollowUpThemes(),
'followup_icons' => \App\Service\FollowUpService::getFollowUpIcons(),
'ctc_completion_series' => $ctc_completion_series,
]);
}
$themes = \App\Service\FollowUpService::getFollowUpThemes();
// On ne vérifie pas ici un thème unique, on boucle sur tous les thèmes plus loin
// Ajout : mesures CTC CityFollowUp pour le graphique séparé
foreach ($stats->getCityFollowUps() as $fu) {
if (preg_match('/^(name|hours|website|address|siret)_count$/', $fu->getName())) {
$ctc_completion_series[$fu->getName()][] = [
'date' => $fu->getDate()->format('Y-m-d'),
'value' => $fu->getMeasure(),
];
}
}
foreach ($ctc_completion_series as &$points) {
usort($points, function($a, $b) {
return strcmp($a['date'], $b['date']);
});
}
unset($points);
return $this->render('admin/followup_graph.html.twig', [
'stats' => $stats,
'completion_tags' => \App\Service\FollowUpService::getFollowUpCompletionTags(),
'followup_labels' => \App\Service\FollowUpService::getFollowUpThemes(),
'followup_icons' => \App\Service\FollowUpService::getFollowUpIcons(),
'ctc_completion_series' => $ctc_completion_series,
]);
}

24
src/Entity/Stats.php
View file

@ -112,6 +112,12 @@ class Stats
#[ORM\OneToMany(mappedBy: 'stats', targetEntity: CityFollowUp::class, cascade: ['persist', 'remove'])]
private Collection $cityFollowUps;
#[ORM\Column(type: 'datetime', nullable: true)]
private ?\DateTime $date_labourage_requested = null;
#[ORM\Column(type: 'datetime', nullable: true)]
private ?\DateTime $date_labourage_done = null;
public function getCTCurlBase(): ?string
{
$base = 'https://complete-tes-commerces.fr/';
@ -630,5 +636,23 @@ class Stats
return $this;
}
public function getDateLabourageRequested(): ?\DateTime
{
return $this->date_labourage_requested;
}
public function setDateLabourageRequested(?\DateTime $date): static
{
$this->date_labourage_requested = $date;
return $this;
}
public function getDateLabourageDone(): ?\DateTime
{
return $this->date_labourage_done;
}
public function setDateLabourageDone(?\DateTime $date): static
{
$this->date_labourage_done = $date;
return $this;
}
}

56
templates/admin/followup_graph.html.twig
View file

@ -136,8 +136,10 @@
'all_types': [type]
} %}
{% endfor %}
<h2 class="mt-4">Comparaison de la complétion par thème</h2>
<canvas id="multiCompletionChart" width="900" height="400"></canvas>
<h2 class="mt-4">Évolution du taux de complétion (CTC - Complète tes commerces)</h2>
<canvas id="ctcCompletionChart" width="900" height="400"></canvas>
<h2 class="mt-4">Données brutes</h2>
<table class="table table-bordered table-striped">
<thead>
@ -172,7 +174,8 @@
<script>
const series = {{ series|json_encode|raw }};
const followupIcons = {{ followup_icons|json_encode|raw }};
const typeLabels = Object.assign({}, {{ followup_labels|json_encode|raw }});
const typeLabels = Object.assign({}, {{ followup_labels|json_encode|raw }});
const ctcCompletionSeries = {% if ctc_completion_series is defined %}{{ ctc_completion_series|json_encode|raw }} {% endif %} | [];
document.addEventListener('DOMContentLoaded', function() {
Object.keys(typeLabels).forEach(function(baseType) {
const countData = (series[baseType + '_count'] || []).map(pt => ({ x: pt.date, y: pt.value }));
@ -268,6 +271,53 @@
}
});
// --- Graphique séparé pour les données CTC ---
if (Object.keys(ctcCompletionSeries).length > 0) {
const ctcDatasets = Object.keys(ctcCompletionSeries).map(function(type) {
return {
label: typeLabels[type.replace('_count','')] || type,
data: ctcCompletionSeries[type].map(pt => ({ x: pt.date, y: pt.value })),
borderColor: 'orange',
backgroundColor: 'rgba(255,165,0,0.1)',
fill: false,
borderDash: [5,3],
datalabels: { display: false }
};
});
const ctcCanvas = document.getElementById('ctcCompletionChart');
if (ctcCanvas) {
new Chart(ctcCanvas, {
type: 'line',
data: { datasets: ctcDatasets },
options: {
parsing: true,
responsive: true,
plugins: {
title: {
display: true,
text: 'Évolution des complétions (CTC)'
},
datalabels: { display: false },
tooltip: {
callbacks: {
title: function(context) {
return context[0].parsed.x ? new Date(context[0].parsed.x).toLocaleString() : '';
},
label: function(context) {
return context.dataset.label + ': ' + context.parsed.y;
}
}
}
},
scales: {
x: { type: 'time', time: { unit: 'day' }, title: { display: true, text: 'Date' } },
y: { beginAtZero: true, title: { display: true, text: 'Nombre' } }
}
},
plugins: [ChartDataLabels]
});
}
}
});
</script>
{% endblock %}

33
templates/admin/stats.html.twig
View file

@ -1166,4 +1166,37 @@ if(dc ){
}
});
</script>
<script>
document.addEventListener('DOMContentLoaded', function() {
const ctcCompletionSeries = {{ ctc_completion_series|json_encode|raw }};
console.log('ctcCompletionSeries',ctcCompletionSeries)
// Exemple d'intégration dans un graphique Chart.js :
// Pour chaque type, ajouter une série CTC si elle existe
Object.keys(ctcCompletionSeries).forEach(function(type) {
const data = ctcCompletionSeries[type].map(pt => ({ x: pt.date, y: pt.value }));
// Ajoute la série au graphique correspondant (ex: name_count, hours_count...)
// À adapter selon l'ID du canvas et la structure du graphique
const canvasId = type.replace('_count','') + 'Chart';
const canvas = document.getElementById(canvasId);
if (!canvas) return;
// On suppose que le graphique existe déjà, on ajoute la série CTC
if (canvas.chart) {
canvas.chart.data.datasets.push({
label: 'CTC (Complète tes commerces)',
data: data,
borderColor: 'orange',
backgroundColor: 'rgba(255,165,0,0.1)',
fill: false,
yAxisID: 'y',
borderDash: [5,3],
datalabels: {
display: false
}
});
canvas.chart.update();
}
});
});
</script>
{% endblock %}