From 3eceb447d955a1afe6d9e6f7e6dc06fa699cb69f Mon Sep 17 00:00:00 2001 From: Carlo Beltrame Date: Thu, 5 Aug 2021 19:12:09 +0200 Subject: [PATCH 1/3] Fix URL not being reported in fetch command help description --- config/services.yaml | 2 +- src/Command/FetchDataCommand.php | 3 +++ tests/Command/FetchDataCommandTest.php | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config/services.yaml b/config/services.yaml index 80068e5..f519a16 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -79,7 +79,7 @@ services: App\Command\FetchDataCommand: arguments: $importDirectory: '%import_data_dir%' - + $url: '%env(PBS_DATA_URL)%' # PBS Services App\Service\PbsApiService: diff --git a/src/Command/FetchDataCommand.php b/src/Command/FetchDataCommand.php index 0519105..67a65b7 100644 --- a/src/Command/FetchDataCommand.php +++ b/src/Command/FetchDataCommand.php @@ -32,13 +32,16 @@ class FetchDataCommand extends StatisticsCommand /** * FetchDataCommand constructor. * @param string $importDirectory + * @param string $url * @param PbsApiService $pbsApiService */ public function __construct( string $importDirectory, + string $url, PbsApiService $pbsApiService ) { $this->targetDir = $importDirectory; + $this->url = $url; $this->pbsApiService = $pbsApiService; parent::__construct(); } diff --git a/tests/Command/FetchDataCommandTest.php b/tests/Command/FetchDataCommandTest.php index 8c4b588..5b92f46 100644 --- a/tests/Command/FetchDataCommandTest.php +++ b/tests/Command/FetchDataCommandTest.php @@ -25,7 +25,7 @@ protected function setUp() { self::bootKernel(); $this->targetDir = self::$kernel->getContainer()->getParameter('import_data_dir'); $this->pbsApiServiceMock = new PbsApiServiceMock(); - $this->command = new FetchDataCommand($this->targetDir, $this->pbsApiServiceMock); + $this->command = new FetchDataCommand($this->targetDir, '', $this->pbsApiServiceMock); $this->commandTester = new CommandTester($this->command); } From 5e993fc568b34b866457bd1103b72804a13e0548 Mon Sep 17 00:00:00 2001 From: Carlo Beltrame Date: Thu, 5 Aug 2021 21:13:27 +0200 Subject: [PATCH 2/3] Fetch geolocations from the normal hitobito JSON API --- src/Command/FetchDataCommand.php | 69 ++++++++++++++++++++++++++++++++ src/Service/PbsApiService.php | 16 ++++++++ 2 files changed, 85 insertions(+) diff --git a/src/Command/FetchDataCommand.php b/src/Command/FetchDataCommand.php index 67a65b7..28a6cc4 100644 --- a/src/Command/FetchDataCommand.php +++ b/src/Command/FetchDataCommand.php @@ -69,6 +69,9 @@ public function execute(InputInterface $input, OutputInterface $output) $this->processPaginatedTable($paginatedTable); } + $this->io->section('Fetching geolocations'); + $this->processGeolocations(); + $this->io->success('Finished fetching all table data successfully'); $this->io->title('Fetch Command Stats'); @@ -201,6 +204,72 @@ private function processPaginatedTable(string $tableName): void $this->processTableStats($filePath, $tableName, $timeElapsed, $totalItemCount, $page - 1); } + /** + * @return void + * @throws Exception + */ + private function processGeolocations(): void + { + $start = microtime(true); + + $filePath = $this->targetDir . '/geolocations.json'; + $groupIds = $this->getAbteilungenToFetch(); + $this->io->text('Will fetch ' . count($groupIds) . ' Abteilungen'); + foreach ($groupIds as $groupId) { + $identifier = 'groups/' . $groupId . '.json'; + $this->io->text('Fetching ' . $identifier); + $result = $this->pbsApiService->getApiData($identifier); + + if ($result->getStatusCode() !== 200) { + $this->io->error([ + 'API call for ' . $identifier . ' failed!', + 'HTTP status code: ' . $result->getStatusCode() + ]); + throw new Exception( + 'Got http status code ' . $result->getStatusCode() . ' from API. Stopped fetching data.' + ); + } + + $content = $result->getContent(); + if (isset($content['groups'][0]['links']['geolocations'])) { + foreach ($content['groups'][0]['links']['geolocations'] as $geolocationId) { + $geolocations = array_filter($content['linked']['geolocations'], function ($geolocation) use ($geolocationId) { + return $geolocation['id'] === $geolocationId; + }); + + if (count($geolocations) === 0) { + continue; + } + + $geolocation = array_values($geolocations)[0]; + $geolocation['group_id'] = $groupId; + $this->appendJsonToFile($filePath, $geolocation); + } + } + } + + $timeElapsed = microtime(true) - $start; + $this->processTableStats($filePath, 'geolocations', $timeElapsed, count($groupIds)); + } + + /** + * Returns the ids of all Abteilung groups that we are allowed to fetch + * @return array + */ + private function getAbteilungenToFetch(): array + { + $fileName = $this->targetDir . '/groups.json'; + $groups = json_decode(file_get_contents($fileName), true); + + $abteilungen = array_filter($groups, function ($group) { + return 'Group::Abteilung' === ($group['type'] ?? ''); + }); + + return array_map(function ($abteilung) { + return $abteilung['id']; + }, $abteilungen); + } + /** * @param $filePath * @param $data diff --git a/src/Service/PbsApiService.php b/src/Service/PbsApiService.php index c557524..32825ad 100644 --- a/src/Service/PbsApiService.php +++ b/src/Service/PbsApiService.php @@ -42,4 +42,20 @@ public function getTableData(string $tableName, int $page = null, int $itemsPerP $additionalHeaders = ['X-Token' => $this->apiKey]; return $this->guzzleWrapper->getJson($endpoint, null, $additionalHeaders); } + + /** + * @param string $tableName + * @param int|null $page + * @param int|null $itemsPerPage + * @return Http\CurlResponse + */ + public function getApiData(string $endpoint, int $page = null, int $itemsPerPage = null) + { + $endpoint = $this->url . '/' . $endpoint; + if ($page !== null && $itemsPerPage !== null) { + $endpoint .= '?page=' . $page . '&size=' . $itemsPerPage; + } + $additionalHeaders = ['X-Token' => $this->apiKey]; + return $this->guzzleWrapper->getJson($endpoint, null, $additionalHeaders); + } } From 99ef4d00abcdd37d7c297b99a15360f9fc69e7cd Mon Sep 17 00:00:00 2001 From: Carlo Beltrame Date: Thu, 5 Aug 2021 21:52:47 +0200 Subject: [PATCH 3/3] Import the geolocations into the database --- src/Command/ImportFromJsonCommand.php | 43 +++++++++++++ src/Entity/GeoLocation.php | 77 ++++++++++++++++++++++++ src/Entity/Group.php | 37 ++++++++++++ src/Migrations/Version20210805192246.php | 36 +++++++++++ src/Repository/GeoLocationRepository.php | 21 +++++++ 5 files changed, 214 insertions(+) create mode 100644 src/Entity/GeoLocation.php create mode 100644 src/Migrations/Version20210805192246.php create mode 100644 src/Repository/GeoLocationRepository.php diff --git a/src/Command/ImportFromJsonCommand.php b/src/Command/ImportFromJsonCommand.php index f4706f2..c183b73 100644 --- a/src/Command/ImportFromJsonCommand.php +++ b/src/Command/ImportFromJsonCommand.php @@ -3,6 +3,7 @@ namespace App\Command; use App\Entity\EventGroup; +use App\Entity\GeoLocation; use App\Entity\YouthSportType; use App\Entity\Camp; use App\Entity\CampState; @@ -108,6 +109,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $this->importParticipations($output); $this->importQualifications($output); $this->importRoles($output); + $this->importGeoLocations($output); $this->em->getConnection()->commit(); @@ -133,6 +135,8 @@ private function cleaningUpEntities(OutputInterface $output) $connection->executeQuery("DELETE FROM midata_event_group"); + $connection->executeQuery("DELETE FROM midata_geo_location"); + $output->writeln("cleaned some entities from the db"); } @@ -758,6 +762,45 @@ private function importRoles(OutputInterface $output) $output->writeln([sprintf('%s rows imported from roles.json', $i)]); } + /** + * @param OutputInterface $output + * @throws Exception + */ + private function importGeoLocations(OutputInterface $output) + { + $start = microtime(true); + $geoLocations = JsonMachine::fromFile(sprintf('%s/geolocations.json', $this->params->get('import_data_dir'))); + $i = 0; + + foreach ($geoLocations as $gl) { + $longitude = floatval($gl['long']); + $latitude = floatval($gl['lat']); + // Comparing float to 0. should be safe, even though comparing floats directly normally isn't + if (0. === $longitude || 0. === $latitude) { + // A parsed value of 0. implies that the string from the JSON was not a float + // because hitobito validates that the coordinates lie in Switzerland + continue; + } + + /** @var Group $abteilung */ + $abteilung = $this->em->getRepository(Group::class)->find($gl['group_id']); + if (!$abteilung) { + continue; + } + + $geoLocation = new GeoLocation(); + $geoLocation->setAbteilung($abteilung); + $geoLocation->setLongitude($longitude); + $geoLocation->setLatitude($latitude); + $this->em->persist($geoLocation); + $this->em->flush(); + $i++; + } + $timeElapsed = microtime(true) - $start; + $this->stats[] = ['geolocations.json', $timeElapsed, $i]; + $output->writeln([sprintf('%s rows imported from geolocations.json', $i)]); + } + public function getStats(): CommandStatistics { $totalItems = 0; diff --git a/src/Entity/GeoLocation.php b/src/Entity/GeoLocation.php new file mode 100644 index 0000000..4e438bc --- /dev/null +++ b/src/Entity/GeoLocation.php @@ -0,0 +1,77 @@ +id; + } + + public function getLongitude(): ?float + { + return $this->longitude; + } + + public function setLongitude(float $longitude): self + { + $this->longitude = $longitude; + + return $this; + } + + public function getLatitude(): ?float + { + return $this->latitude; + } + + public function setLatitude(float $latitude): self + { + $this->latitude = $latitude; + + return $this; + } + + public function getAbteilung(): ?Group + { + return $this->abteilung; + } + + public function setAbteilung(?Group $abteilung): self + { + $this->abteilung = $abteilung; + + return $this; + } +} diff --git a/src/Entity/Group.php b/src/Entity/Group.php index 636c9dc..a9b5336 100644 --- a/src/Entity/Group.php +++ b/src/Entity/Group.php @@ -4,6 +4,7 @@ use DateTimeImmutable; use Doctrine\Common\Collections\ArrayCollection; +use Doctrine\Common\Collections\Collection; use Doctrine\ORM\Mapping as ORM; /** @@ -77,9 +78,15 @@ class Group */ private $personRoles; + /** + * @ORM\OneToMany(targetEntity=GeoLocation::class, mappedBy="abteilung", orphanRemoval=true) + */ + private $geoLocations; + public function __construct() { $this->events = new ArrayCollection(); + $this->geoLocations = new ArrayCollection(); } /** @@ -214,4 +221,34 @@ public function __toString() { return (string)$this->id; } + + /** + * @return Collection|GeoLocation[] + */ + public function getGeoLocations(): Collection + { + return $this->geoLocations; + } + + public function addGeoLocation(GeoLocation $geoLocation): self + { + if (!$this->geoLocations->contains($geoLocation)) { + $this->geoLocations[] = $geoLocation; + $geoLocation->setAbteilung($this); + } + + return $this; + } + + public function removeGeoLocation(GeoLocation $geoLocation): self + { + if ($this->geoLocations->removeElement($geoLocation)) { + // set the owning side to null (unless already changed) + if ($geoLocation->getAbteilung() === $this) { + $geoLocation->setAbteilung(null); + } + } + + return $this; + } } diff --git a/src/Migrations/Version20210805192246.php b/src/Migrations/Version20210805192246.php new file mode 100644 index 0000000..9c3b1a0 --- /dev/null +++ b/src/Migrations/Version20210805192246.php @@ -0,0 +1,36 @@ +addSql('CREATE SEQUENCE midata_geo_location_id_seq INCREMENT BY 1 MINVALUE 1 START 1'); + $this->addSql('CREATE TABLE midata_geo_location (id INT NOT NULL, abteilung_id INT NOT NULL, longitude DOUBLE PRECISION NOT NULL, latitude DOUBLE PRECISION NOT NULL, PRIMARY KEY(id))'); + $this->addSql('CREATE INDEX IDX_B027FE6AE862333F ON midata_geo_location (abteilung_id)'); + $this->addSql('ALTER TABLE midata_geo_location ADD CONSTRAINT FK_B027FE6AE862333F FOREIGN KEY (abteilung_id) REFERENCES midata_group (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + } + + public function down(Schema $schema) : void + { + // this down() migration is auto-generated, please modify it to your needs + $this->addSql('CREATE SCHEMA public'); + $this->addSql('DROP SEQUENCE midata_geo_location_id_seq CASCADE'); + $this->addSql('DROP TABLE midata_geo_location'); + } +} diff --git a/src/Repository/GeoLocationRepository.php b/src/Repository/GeoLocationRepository.php new file mode 100644 index 0000000..7a37794 --- /dev/null +++ b/src/Repository/GeoLocationRepository.php @@ -0,0 +1,21 @@ +