mirror of
https://forge.chapril.org/tykayn/osm-commerces
synced 2025-11-19 23:00:36 +01:00
up commande labourage queue
This commit is contained in:
Tykayn
tykayn
parent
ca0ec580f5
commit
1345cc903b
9 changed files with 532 additions and 441 deletions
130
src/Command/ImportCityFollowupFromCTCCommand.php
Normal file
130
src/Command/ImportCityFollowupFromCTCCommand.php
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
<?php
|
||||
|
||||
namespace App\Command;
|
||||
|
||||
use App\Entity\Stats;
|
||||
use App\Entity\CityFollowUp;
|
||||
use Doctrine\ORM\EntityManagerInterface;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Console\Style\SymfonyStyle;
|
||||
|
||||
#[AsCommand(
|
||||
name: 'app:import-cityfollowup-ctc',
|
||||
description: 'Importe les CityFollowUp à partir du JSON DailyStats d\'une ville (Complète tes commerces)'
|
||||
)]
|
||||
class ImportCityFollowupFromCTCCommand extends Command
|
||||
{
|
||||
public function __construct(private EntityManagerInterface $entityManager)
|
||||
{
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void
|
||||
{
|
||||
$this
|
||||
->addArgument('insee_code', InputArgument::REQUIRED, 'Code INSEE de la ville')
|
||||
->addOption('url', null, InputOption::VALUE_OPTIONAL, 'URL CTC de la ville (sinon auto-déduit)')
|
||||
->addOption('dry-run', null, InputOption::VALUE_NONE, 'Simule l\'import sans rien écrire');
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$io = new SymfonyStyle($input, $output);
|
||||
$insee = $input->getArgument('insee_code');
|
||||
$url = $input->getOption('url');
|
||||
$dryRun = $input->getOption('dry-run');
|
||||
|
||||
$stats = $this->entityManager->getRepository(Stats::class)->findOneBy(['zone' => $insee]);
|
||||
if (!$stats) {
|
||||
$io->error("Aucune stats trouvée pour le code INSEE $insee");
|
||||
return Command::FAILURE;
|
||||
}
|
||||
if (!$url) {
|
||||
$url = $stats->getCTCurlBase() . '_dailystats.json';
|
||||
}
|
||||
$io->title("Import des CityFollowUp depuis $url");
|
||||
// --- Gestion explicite des erreurs HTTP (404, etc.) ---
|
||||
$context = stream_context_create([
|
||||
'http' => [
|
||||
'ignore_errors' => true,
|
||||
'timeout' => 10,
|
||||
]
|
||||
]);
|
||||
$json = @file_get_contents($url, false, $context);
|
||||
$http_response_header = $http_response_header ?? [];
|
||||
$httpCode = null;
|
||||
foreach ($http_response_header as $header) {
|
||||
if (preg_match('#^HTTP/\\d+\\.\\d+ (\\d{3})#', $header, $m)) {
|
||||
$httpCode = (int)$m[1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if ($json === false || $httpCode === 404) {
|
||||
$io->error("Impossible de télécharger le JSON DailyStats depuis $url (erreur HTTP $httpCode)");
|
||||
return Command::FAILURE;
|
||||
}
|
||||
$data = json_decode($json, true);
|
||||
if (!is_array($data)) {
|
||||
$io->error("Le JSON n'est pas un tableau valide");
|
||||
return Command::FAILURE;
|
||||
}
|
||||
$types = [
|
||||
'name' => ['field' => 'no_name', 'label' => 'name'],
|
||||
'hours' => ['field' => 'no_hours', 'label' => 'hours'],
|
||||
'website' => ['field' => 'no_website', 'label' => 'website'],
|
||||
'address' => ['field' => 'no_address', 'label' => 'address'],
|
||||
'siret' => ['field' => 'no_siret', 'label' => 'siret'],
|
||||
];
|
||||
$created = 0;
|
||||
$skipped = 0;
|
||||
$createdEntities = [];
|
||||
foreach ($data as $row) {
|
||||
$date = isset($row['date']) ? new \DateTime($row['date']) : null;
|
||||
if (!$date) continue;
|
||||
$total = $row['total'] ?? null;
|
||||
if (!$total) continue;
|
||||
foreach ($types as $type => $info) {
|
||||
$field = $info['field'];
|
||||
if (!isset($row[$field])) continue;
|
||||
$measure = $total - $row[$field]; // nombre d'objets complets pour ce champ
|
||||
$name = $type . '_count';
|
||||
// Vérifier doublon (même stats, même nom, même date)
|
||||
$existing = $this->entityManager->getRepository(CityFollowUp::class)->findOneBy([
|
||||
'stats' => $stats,
|
||||
'name' => $name,
|
||||
'date' => $date,
|
||||
]);
|
||||
if ($existing) {
|
||||
$skipped++;
|
||||
continue;
|
||||
}
|
||||
$cfu = new CityFollowUp();
|
||||
$cfu->setStats($stats)
|
||||
->setName($name)
|
||||
->setDate($date)
|
||||
->setMeasure($measure);
|
||||
if (!$dryRun) {
|
||||
$this->entityManager->persist($cfu);
|
||||
$createdEntities[] = $cfu;
|
||||
}
|
||||
$created++;
|
||||
}
|
||||
}
|
||||
if (!$dryRun) {
|
||||
$this->entityManager->flush();
|
||||
// Vérification explicite du lien
|
||||
foreach ($createdEntities as $cfu) {
|
||||
if (!$cfu->getStats() || $cfu->getStats()->getId() !== $stats->getId()) {
|
||||
$io->warning('CityFollowUp non lié correctement à Stats ID ' . $stats->getId() . ' (ID CityFollowUp: ' . $cfu->getId() . ')');
|
||||
}
|
||||
}
|
||||
}
|
||||
$io->success("$created CityFollowUp créés, $skipped doublons ignorés.");
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
}
|
||||
150
src/Command/ProcessLabourageQueueCommand.php
Normal file
150
src/Command/ProcessLabourageQueueCommand.php
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
<?php
|
||||
|
||||
namespace App\Command;
|
||||
|
||||
use App\Entity\Stats;
|
||||
use App\Entity\Place;
|
||||
use Doctrine\ORM\EntityManagerInterface;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Console\Style\SymfonyStyle;
|
||||
use App\Service\Motocultrice;
|
||||
use App\Service\FollowUpService;
|
||||
|
||||
#[AsCommand(
|
||||
name: 'app:process-labourage-queue',
|
||||
description: 'Traite la file d\'attente de labourage différé des villes (cron)'
|
||||
)]
|
||||
class ProcessLabourageQueueCommand extends Command
|
||||
{
|
||||
public function __construct(
|
||||
private EntityManagerInterface $entityManager,
|
||||
private Motocultrice $motocultrice,
|
||||
private FollowUpService $followUpService
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$io = new SymfonyStyle($input, $output);
|
||||
// Sélectionner la Stats à traiter (date_labourage_requested la plus ancienne, non traitée ou à refaire)
|
||||
$stats = $this->entityManager->getRepository(Stats::class)
|
||||
->createQueryBuilder('s')
|
||||
->where('s.date_labourage_requested IS NOT NULL')
|
||||
->andWhere('s.date_labourage_done IS NULL OR s.date_labourage_done < s.date_labourage_requested')
|
||||
->andWhere('s.zone != :global_zone')
|
||||
->setParameter('global_zone', '00000')
|
||||
->orderBy('s.date_labourage_requested', 'ASC')
|
||||
->setMaxResults(1)
|
||||
->getQuery()
|
||||
->getOneOrNullResult();
|
||||
if (!$stats) {
|
||||
// 1. Villes jamais labourées (date_labourage_done NULL, hors 00000)
|
||||
$stats = $this->entityManager->getRepository(Stats::class)
|
||||
->createQueryBuilder('s')
|
||||
->where('s.zone != :global_zone')
|
||||
->andWhere('s.date_labourage_done IS NULL')
|
||||
->setParameter('global_zone', '00000')
|
||||
->orderBy('s.date_modified', 'ASC')
|
||||
->setMaxResults(1)
|
||||
->getQuery()
|
||||
->getOneOrNullResult();
|
||||
if ($stats) {
|
||||
$io->note('Aucune ville en attente, on traite en priorité une ville jamais labourée : ' . $stats->getName() . ' (' . $stats->getZone() . ')');
|
||||
$stats->setDateLabourageRequested(new \DateTime());
|
||||
$this->entityManager->persist($stats);
|
||||
$this->entityManager->flush();
|
||||
} else {
|
||||
// 2. Ville la plus anciennement modifiée (hors 00000)
|
||||
$stats = $this->entityManager->getRepository(Stats::class)
|
||||
->createQueryBuilder('s')
|
||||
->where('s.zone != :global_zone')
|
||||
->setParameter('global_zone', '00000')
|
||||
->orderBy('s.date_modified', 'ASC')
|
||||
->setMaxResults(1)
|
||||
->getQuery()
|
||||
->getOneOrNullResult();
|
||||
if (!$stats) {
|
||||
$io->success('Aucune ville à traiter.');
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
$io->note('Aucune ville en attente, on demande le labourage de la ville la plus anciennement modifiée : ' . $stats->getName() . ' (' . $stats->getZone() . ')');
|
||||
$stats->setDateLabourageRequested(new \DateTime());
|
||||
$this->entityManager->persist($stats);
|
||||
$this->entityManager->flush();
|
||||
}
|
||||
}
|
||||
$io->section('Traitement de la ville : ' . $stats->getName() . ' (' . $stats->getZone() . ')');
|
||||
// Vérifier la RAM disponible (>= 1 Go)
|
||||
$meminfo = @file_get_contents('/proc/meminfo');
|
||||
$ram_ok = false;
|
||||
if ($meminfo !== false && preg_match('/^MemAvailable:\s+(\d+)/m', $meminfo, $matches)) {
|
||||
$mem_kb = (int)$matches[1];
|
||||
$ram_ok = ($mem_kb >= 1024 * 1024); // 1 Go
|
||||
}
|
||||
if (!$ram_ok) {
|
||||
$io->warning('RAM insuffisante, on attend le prochain cron.');
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
// Effectuer le labourage complet (reprendre la logique de création/màj des objets Place)
|
||||
$io->info('RAM suffisante, lancement du labourage...');
|
||||
$places_overpass = $this->motocultrice->labourer($stats->getZone());
|
||||
$processedCount = 0;
|
||||
$updatedCount = 0;
|
||||
$existingPlacesQuery = $this->entityManager->getRepository(Place::class)
|
||||
->createQueryBuilder('p')
|
||||
->select('p.osmId, p.osm_kind, p.id')
|
||||
->where('p.zip_code = :zip_code')
|
||||
->setParameter('zip_code', $stats->getZone())
|
||||
->getQuery();
|
||||
$existingPlacesResult = $existingPlacesQuery->getResult();
|
||||
$placesByOsmKey = [];
|
||||
foreach ($existingPlacesResult as $placeData) {
|
||||
$osmKey = $placeData['osm_kind'] . '_' . $placeData['osmId'];
|
||||
$placesByOsmKey[$osmKey] = $placeData['id'];
|
||||
}
|
||||
foreach ($places_overpass as $placeData) {
|
||||
$osmKey = $placeData['type'] . '_' . $placeData['id'];
|
||||
$existingPlaceId = $placesByOsmKey[$osmKey] ?? null;
|
||||
if (!$existingPlaceId) {
|
||||
$place = new Place();
|
||||
$place->setOsmId($placeData['id'])
|
||||
->setOsmKind($placeData['type'])
|
||||
->setZipCode($stats->getZone())
|
||||
->setUuidForUrl($this->motocultrice->uuid_create())
|
||||
->setModifiedDate(new \DateTime())
|
||||
->setStats($stats)
|
||||
->setDead(false)
|
||||
->setOptedOut(false)
|
||||
->setMainTag($this->motocultrice->find_main_tag($placeData['tags']) ?? '')
|
||||
->setStreet($this->motocultrice->find_street($placeData['tags']) ?? '')
|
||||
->setHousenumber($this->motocultrice->find_housenumber($placeData['tags']) ?? '')
|
||||
->setSiret($this->motocultrice->find_siret($placeData['tags']) ?? '')
|
||||
->setAskedHumainsSupport(false)
|
||||
->setLastContactAttemptDate(null)
|
||||
->setPlaceCount(0);
|
||||
$place->update_place_from_overpass_data($placeData);
|
||||
$this->entityManager->persist($place);
|
||||
$stats->addPlace($place);
|
||||
$processedCount++;
|
||||
} else {
|
||||
$existingPlace = $this->entityManager->getRepository(Place::class)->find($existingPlaceId);
|
||||
if ($existingPlace) {
|
||||
$existingPlace->setDead(false);
|
||||
$existingPlace->update_place_from_overpass_data($placeData);
|
||||
$stats->addPlace($existingPlace);
|
||||
$this->entityManager->persist($existingPlace);
|
||||
$updatedCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
$stats->setDateLabourageDone(new \DateTime());
|
||||
$this->entityManager->persist($stats);
|
||||
$this->entityManager->flush();
|
||||
$io->success("Labourage terminé : $processedCount nouveaux lieux, $updatedCount lieux mis à jour.");
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue