diff --git a/app/Concerns/Import/HasPlace.php b/app/Concerns/Import/HasPlace.php new file mode 100644 index 0000000..4fe3361 --- /dev/null +++ b/app/Concerns/Import/HasPlace.php @@ -0,0 +1,107 @@ +old_ids)->each( + fn (int $oldId) => $countries->put($oldId, $country->id) + ); + }); + + return $countries; + } + + protected function getCounties(): Collection + { + return County::pluck('id', 'old_id'); + } + + protected function getLocalities(): Collection + { + $localities = collect(); + + Locality::query() + ->whereNotNull('old_ids') + ->each(function (Locality $locality) use ($localities) { + collect($locality->old_ids)->each( + fn (int $oldId) => $localities->put($oldId, $locality->id) + ); + }); + + return $localities; + } + + protected function getPlace(stdClass $row): ?array + { + if (blank($this->countries)) { + $this->countries = $this->getCountries(); + } + + if (blank($this->counties)) { + $this->counties = $this->getCounties(); + } + + if (blank($this->localities)) { + $this->localities = $this->getLocalities(); + } + + $place = [ + 'country_id' => $this->countries->get($row->CountryId), + 'county_id' => $this->counties->get($row->CountyId), + 'locality_id' => $this->localities->get($row->LocalityId), + ]; + + $validation = Validator::make($place, [ + 'country_id' => ['required_without:county_id,locality_id'], + 'county_id' => ['required_without:country_id', 'required_with:locality_id'], + 'locality_id' => ['required_without:country_id', 'required_with:county_id'], + ]); + + if ($validation->fails()) { + if ($place['county_id'] === 403 && blank($place['locality_id'])) { + // TODO: Date doar pe București, TBD + // old ballot ids 62, 63, 64, 65, 69, 69, 71, 72, 73, 74 + } elseif ( + blank($place['locality_id']) && + $row->LocalityId >= 64413 && + $row->LocalityId <= 64497 + ) { + // TODO: Toate Localitatile Din Judet + // old ballot ids 71, 72, 73, 74 + } else { + logger()->error('Could not determine location.', [ + 'BallotId' => $row->BallotId, + 'TurnoutId' => $row->Id, + 'CountyId' => $row->CountyId, + 'LocalityId' => $row->LocalityId, + 'locality_id' => $place['locality_id'], + ]); + + return null; + } + } + + return $place; + } +} diff --git a/app/Console/Commands/Import/ImportCommand.php b/app/Console/Commands/Import/ImportCommand.php index b4c6919..5437f07 100644 --- a/app/Console/Commands/Import/ImportCommand.php +++ b/app/Console/Commands/Import/ImportCommand.php @@ -41,10 +41,15 @@ public function handle(): int $this->call(ImportOldIdsCommand::class, [ '--force' => $this->option('force'), ]); + $this->call(ImportTurnoutsCommand::class, [ '--force' => $this->option('force'), ]); + $this->call(ImportPartiesCandidatesCommand::class, [ + '--force' => $this->option('force'), + ]); + return self::SUCCESS; } } diff --git a/app/Console/Commands/Import/ImportElectionsCommand.php b/app/Console/Commands/Import/ImportElectionsCommand.php index 6bedc04..95e6ddd 100644 --- a/app/Console/Commands/Import/ImportElectionsCommand.php +++ b/app/Console/Commands/Import/ImportElectionsCommand.php @@ -56,6 +56,21 @@ public function handle(): int }, 'date' => $row->Date, 'is_live' => false, + 'has_lists' => match ($row->BallotType) { + // Referendum = 0, + // President = 1, + // Senate = 2, + // House = 3, + // LocalCouncil = 4, + // CountyCouncil = 5, + // Mayor = 6, + // EuropeanParliament = 7, + // CountyCouncilPresident = 8, + // CapitalCityMayor = 9, + // CapitalCityCouncil = 10, + 0, 1, 6, 8, 9 => false, + 2, 3, 4, 5, 7, 10 => true, + }, 'old_id' => $row->BallotId, ]); }, (int) $this->option('chunk')); diff --git a/app/Console/Commands/Import/ImportOldIdsCommand.php b/app/Console/Commands/Import/ImportOldIdsCommand.php index 5bddd55..5ae9cb7 100644 --- a/app/Console/Commands/Import/ImportOldIdsCommand.php +++ b/app/Console/Commands/Import/ImportOldIdsCommand.php @@ -55,10 +55,15 @@ protected function importCountryIds(): void ); $query->each(function (stdClass $row) { - $country = Country::search($row->Name)->first(); + $name = match ($row->Name) { + 'Geneva' => 'Elveția', + default => $row->Name, + }; + + $country = Country::search($name)->first(); if (blank($country)) { - logger()->error("Country not found: {$row->Name}"); + logger()->error("Country not found: {$name}"); return; } @@ -110,7 +115,7 @@ protected function importLocalityIds(): void { $query = $this->db ->table('localities') - ->orderBy('localities.Siruta'); + ->orderBy('localities.LocalityId'); $this->createProgressBar( 'Importing locality IDs...', @@ -120,12 +125,6 @@ protected function importLocalityIds(): void $counties = County::pluck('id', 'old_id'); $query->each(function (stdClass $row) use ($counties) { - // $siruta = match ($row->Siruta) { - // 116921 => 61069, // Băneasa, Constanța - // 713, 21469 => 9280, // Fântânele, Arad - // default => $row->Siruta - // }; - if ($row->Siruta === 0) { $locality = $this->searchLocalities($row->Name, $counties->get($row->CountyId)); } else { @@ -134,10 +133,10 @@ protected function importLocalityIds(): void ->firstOr(fn () => $this->searchLocalities($row->Name, $counties->get($row->CountyId))); } - logger()->info("{$row->LocalityId} | {$row->Name} | Siruta: {$row->Siruta} => " . $locality?->name ?? 'NULL'); + // logger()->info("{$row->LocalityId} | {$row->Name} | Siruta: {$row->Siruta} => " . $locality?->name ?? 'NULL'); if (blank($locality)) { - logger()->error("Locality not found: {$row->Name}"); + logger()->error("Locality not found: {$row->LocalityId} | {$row->Name} | Siruta: {$row->Siruta}"); return; } @@ -157,8 +156,17 @@ protected function importLocalityIds(): void protected function searchLocalities(string $name, int $county_id): ?Locality { + $name = match (true) { + $county_id === 118 && $name === 'Pescari' => 'Coronini', + $county_id === 136 && $name === 'Basarabi' => 'Murfatlar', + $county_id === 207 && $name === 'Unirea' => 'General Berthelot', + default => $name, + }; + return Locality::search($name) ->where('county_id', $county_id) + ->get() + ->sortBy('parent_id') ->first(); } } diff --git a/app/Console/Commands/Import/ImportPartiesCandidatesCommand.php b/app/Console/Commands/Import/ImportPartiesCandidatesCommand.php new file mode 100644 index 0000000..85e7cbf --- /dev/null +++ b/app/Console/Commands/Import/ImportPartiesCandidatesCommand.php @@ -0,0 +1,258 @@ +confirmToProceed()) { + return static::FAILURE; + } + + Schema::withoutForeignKeyConstraints(function () { + Candidate::truncate(); + Party::truncate(); + Vote::truncate(); + }); + + $partyList = $this->getPartyList(); + + Election::query() + ->withoutGlobalScopes() + ->each(function (Election $election) use ($partyList) { + $index = 1; + + $this->importPartiesAndCandidates($election, $partyList); + $this->importVotes($election, $index); + }); + + // $this->parse + + return static::SUCCESS; + } + + protected function getPartyList(): Collection + { + return $this->db + ->table('parties') + ->orderBy('parties.Id') + ->get() + ->keyBy('Id') + ->each(function (stdClass $row) { + if (filled($row->LogoUrl)) { + $path = 'parties/' . $row->Id . '.' . pathinfo($row->LogoUrl, \PATHINFO_EXTENSION); + + if (! Storage::disk('local')->exists($path)) { + Storage::disk('local') + ->put($path, file_get_contents($row->LogoUrl)); + } + } + + return $row; + }); + } + + protected function importPartiesAndCandidates(Election $election, Collection $partyList): void + { + $this->parties = collect(); + $this->candidates = collect(); + + $query = $this->db + ->table('candidateresults') + ->select(['Name', 'ShortName', 'PartyName', 'PartyId']) + ->distinct() + ->where('BallotId', $election->old_id) + ->whereIn('Division', [3, 4]) + ->orderBy('Name'); + + $this->createProgressBar( + "Importing candidates and parties for election #{$election->id}...", + $query->count() + ); + + $query->each(function (stdClass $row) use ($election, $partyList) { + $candidate = $party = null; + + if (filled($row->PartyId)) { + $party = $this->parties->get($row->PartyId, function () use ($election, $row, $partyList) { + $item = $partyList->get($row->PartyId); + + /** @var Party */ + $p = $election->parties()->create([ + 'name' => $item->Name, + 'acronym' => $item->ShortName, + 'color' => $item->Color, + ]); + + if (filled($item->LogoUrl)) { + $ext = pathinfo($item->LogoUrl, \PATHINFO_EXTENSION); + $p->addMediaFromDisk("{$item->Id}.{$ext}", 'local'); + } + + $this->parties->put($row->PartyId, $p->only('id', 'name')); + + return $p; + }); + } + + $candidateName = $row->Name ?: $row->ShortName; + + if (Str::slug($candidateName) !== Str::slug(data_get($party, 'name'))) { + $candidate = $election->candidates()->create([ + 'name' => $candidateName, + 'party_id' => data_get($party, 'id'), + ]); + + $this->candidates->push($candidate->only('id', 'name', 'party_id')); + } + + $this->progressBar->advance(); + }, (int) $this->option('chunk')); + + $this->finishProgressBar("Imported candidates and parties for election #{$election->id}."); + } + + protected function importVotes(Election $election, int &$index): void + { + $query = $this->db + ->table('candidateresults') + ->where('BallotId', $election->old_id) + ->whereIn('Division', [3, 4]) + ->orderBy('candidateresults.Id') + ->leftJoin('localities', 'candidateresults.LocalityId', '=', 'localities.LocalityId') + ->leftJoin('countries', 'candidateresults.CountryId', '=', 'countries.Id') + ->select([ + 'candidateresults.*', + 'localities.Name as LocalityName', + 'countries.Name as CountryName', + ]); + + $this->createProgressBar( + "Importing votes for election #{$election->id}...", + $query->count() + ); + + $query->chunk((int) $this->option('chunk'), function (Collection $rows) use ($election, &$index) { + // TODO: implement + if ($election->type->is(ElectionType::REFERENDUM)) { + return; + } + + Vote::insert( + $rows + ->map(function (stdClass $row) use ($election, &$index) { + $place = $this->getPlace($row); + + $section = $index++; + + if (blank($place)) { + return null; + } + + $partyName = $row->PartyName; + $candidateName = $row->Name ?: $row->ShortName; + + if ($election->has_lists) { + $votable = filled($row->PartyId) + ? $this->getParty($partyName) + : $this->getCandidate($candidateName); + } else { + $votable = $this->getCandidate($candidateName); + } + + if (blank($votable)) { + throw new Exception("Votable id not found for {$candidateName}."); + } + + return [ + 'election_id' => $election->id, + 'section' => $section, + 'part' => Part::FINAL, + + 'votes' => $row->Votes, + ...$votable, + ...$place, + ]; + }) + ->filter() + ->toArray() + ); + + $this->progressBar->advance($rows->count()); + }); + + $this->finishProgressBar("Imported votes for election #{$election->id}."); + } + + protected function getParty(string $name): ?array + { + $party = $this->parties + ->firstWhere(fn (array $party) => Str::slug($party['name']) === Str::slug($name)); + + if (blank($party)) { + return null; + } + + return [ + 'votable_type' => (new Party)->getMorphClass(), + 'votable_id' => $party['id'], + ]; + } + + protected function getCandidate(string $name): ?array + { + $candidate = $this->candidates + ->firstWhere(fn (array $candidate) => Str::slug($candidate['name']) === Str::slug($name)); + + if (blank($candidate)) { + return null; + } + + return [ + 'votable_type' => (new Candidate)->getMorphClass(), + 'votable_id' => $candidate['id'], + ]; + } +} diff --git a/app/Console/Commands/Import/ImportPrepareCommand.php b/app/Console/Commands/Import/ImportPrepareCommand.php index 2136cc8..9960b62 100644 --- a/app/Console/Commands/Import/ImportPrepareCommand.php +++ b/app/Console/Commands/Import/ImportPrepareCommand.php @@ -4,6 +4,7 @@ namespace App\Console\Commands\Import; +use App\Models\User; use Illuminate\Database\Console\Migrations\FreshCommand; class ImportPrepareCommand extends Command @@ -36,6 +37,10 @@ public function handle(): int $this->callSilent(FreshCommand::class); $this->progressBar->advance(); + User::factory(['email' => 'admin@example.com']) + ->admin() + ->create(); + $this->finishProgressBar('Removed old data'); return static::SUCCESS; diff --git a/app/Console/Commands/Import/ImportTurnoutsCommand.php b/app/Console/Commands/Import/ImportTurnoutsCommand.php index 58e51da..2c6c2dd 100644 --- a/app/Console/Commands/Import/ImportTurnoutsCommand.php +++ b/app/Console/Commands/Import/ImportTurnoutsCommand.php @@ -4,18 +4,20 @@ namespace App\Console\Commands\Import; +use App\Concerns\Import\HasPlace; +use App\Enums\Part; use App\Models\Country; -use App\Models\County; use App\Models\Election; use App\Models\Locality; use App\Models\Record; use App\Models\Turnout; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Validator; use stdClass; class ImportTurnoutsCommand extends Command { + use HasPlace; + /** * The name and signature of the console command. * @@ -32,12 +34,6 @@ class ImportTurnoutsCommand extends Command */ protected $description = 'Import turnouts from the old database.'; - protected ?Collection $countries = null; - - protected ?Collection $counties = null; - - protected ?Collection $localities = null; - /** * Execute the console command. */ @@ -64,10 +60,6 @@ public function handle(): int $query->count() ); - $this->countries = $this->getCountries(); - $this->counties = County::pluck('id', 'old_id'); - $this->localities = $this->getLocalities(); - Election::all()->each(function (Election $election) use ($query) { $index = 1; @@ -80,32 +72,50 @@ public function handle(): int $rows->each(function (stdClass $row) use ($election, &$index, &$turnouts, &$records) { $place = $this->getPlace($row); + $section = $index++; + if (blank($place)) { return; } $turnouts[] = [ 'election_id' => $election->id, - 'section' => $index++, + 'section' => $section, 'initial_permanent' => $row->EligibleVoters, 'initial_complement' => 0, - 'permanent' => $row->PermanentListsVotes, - 'complement' => $row->SpecialListsVotes, - 'supplement' => $row->SuplimentaryVotes, - 'mobile' => 0, + 'permanent' => $row->TotalVotes, + 'complement' => 0, //$row->SpecialListsVotes, + 'supplement' => 0, //$row->SuplimentaryVotes, + 'mobile' => 0, //max($row->VotesByMail, $row->CorrespondenceVotes), ...$place, ]; $records[] = [ + 'election_id' => $election->id, + 'section' => $section, + 'part' => Part::FINAL, + + 'eligible_voters_permanent' => $row->EligibleVoters, + 'eligible_voters_special' => 0, + + 'present_voters_permanent' => $row->TotalVotes, + 'present_voters_special' => 0, + 'present_voters_supliment' => 0, + + 'papers_received' => 0, + 'papers_unused' => 0, + 'votes_valid' => $row->ValidVotes, + 'votes_null' => $row->NullVotes, + ...$place, ]; }); Turnout::insert($turnouts); - // Record::insert($records); + Record::insert($records); $this->progressBar->advance($rows->count()); }); @@ -115,61 +125,4 @@ public function handle(): int return static::SUCCESS; } - - protected function getCountries(): Collection - { - $countries = collect(); - - Country::each(function (Country $country) use ($countries) { - collect($country->old_ids)->each( - fn (int $oldId) => $countries->put($oldId, $country->id) - ); - }); - - return $countries; - } - - protected function getLocalities(): Collection - { - $localities = collect(); - - Locality::query() - ->whereNotNull('old_ids') - ->each(function (Locality $locality) use ($localities) { - collect($locality->old_ids)->each( - fn (int $oldId) => $localities->put($oldId, $locality->id) - ); - }); - - return $localities; - } - - protected function getPlace(stdClass $row): ?array - { - $place = [ - 'country_id' => $this->countries->get($row->CountryId), - 'county_id' => $this->counties->get($row->CountyId), - 'locality_id' => $this->localities->get($row->LocalityId), - ]; - - $validation = Validator::make($place, [ - 'country_id' => ['required_without:county_id,locality_id'], - 'county_id' => ['required_without:country_id', 'required_with:locality_id'], - 'locality_id' => ['required_without:country_id', 'required_with:county_id'], - ]); - - if ($validation->fails()) { - logger()->error('Could not determine location.', [ - 'BallotId' => $row->BallotId, - 'TurnoutId' => $row->Id, - 'CountyId' => $row->CountyId, - 'LocalityId' => $row->LocalityId, - 'locality_id' => $place['locality_id'], - ]); - - return null; - } - - return $place; - } } diff --git a/app/Filament/Admin/Resources/ElectionResource.php b/app/Filament/Admin/Resources/ElectionResource.php index 87bf658..066a793 100644 --- a/app/Filament/Admin/Resources/ElectionResource.php +++ b/app/Filament/Admin/Resources/ElectionResource.php @@ -93,6 +93,10 @@ public static function form(Form $form): Form ->label(__('app.field.is_live')) ->default(false), + Toggle::make('has_lists') + ->label(__('app.field.has_lists')) + ->default(false), + Select::make('properties.default_tab') ->label(__('app.field.default_tab')) ->options(DefaultElectionPage::options()) diff --git a/app/Models/Election.php b/app/Models/Election.php index a177bc1..62a2909 100644 --- a/app/Models/Election.php +++ b/app/Models/Election.php @@ -30,6 +30,7 @@ class Election extends Model implements HasName, HasAvatar 'slug', 'date', 'is_live', + 'has_lists', 'properties', 'old_id', ]; @@ -41,6 +42,7 @@ protected function casts(): array 'date' => 'date', 'year' => 'int', 'is_live' => 'boolean', + 'has_lists' => 'boolean', 'properties' => 'collection', 'old_id' => 'int', ]; @@ -69,6 +71,21 @@ public function voteMonitorStats(): HasMany return $this->hasMany(VoteMonitorStat::class); } + public function parties(): HasMany + { + return $this->hasMany(Party::class); + } + + public function candidates(): HasMany + { + return $this->hasMany(Candidate::class); + } + + public function votes(): HasMany + { + return $this->hasMany(Vote::class); + } + public function scopeWhereLive(Builder $query): Builder { return $query->where('is_live', true); diff --git a/app/Models/Locality.php b/app/Models/Locality.php index 72a3cf3..a6cf99f 100644 --- a/app/Models/Locality.php +++ b/app/Models/Locality.php @@ -82,6 +82,7 @@ public function toSearchableArray(): array 'name' => $this->name, 'county' => $this->county->name, 'county_id' => (string) $this->county_id, + 'parent_id' => (string) $this->parent_id ?: null, ]; } @@ -106,6 +107,11 @@ public static function getTypesenseModelSettings(): array 'name' => 'county_id', 'type' => 'string', ], + [ + 'name' => 'parent_id', + 'type' => 'string', + 'optional' => true, + ], ], ], 'search-parameters' => [ diff --git a/database/migrations/0001_01_02_000001_create_election_tables.php b/database/migrations/0001_01_02_000001_create_election_tables.php index f105130..8dd066d 100644 --- a/database/migrations/0001_01_02_000001_create_election_tables.php +++ b/database/migrations/0001_01_02_000001_create_election_tables.php @@ -19,6 +19,7 @@ public function up(): void $table->date('date'); $table->year('year')->storedAs('(YEAR(date))'); $table->boolean('is_live'); + $table->boolean('has_lists')->default(false); $table->json('properties')->nullable(); $table->timestamps(); diff --git a/database/migrations/0001_01_02_000010_create_parties_table.php b/database/migrations/0001_01_02_000010_create_parties_table.php index d169be1..1e97008 100644 --- a/database/migrations/0001_01_02_000010_create_parties_table.php +++ b/database/migrations/0001_01_02_000010_create_parties_table.php @@ -24,7 +24,6 @@ public function up(): void $table->timestamps(); $table->unique(['name', 'election_id']); - $table->unique(['color', 'election_id']); $table->unique(['acronym', 'election_id']); }); } diff --git a/database/migrations/0001_01_02_000020_create_candidates_table.php b/database/migrations/0001_01_02_000020_create_candidates_table.php index 6fd77fa..5843a52 100644 --- a/database/migrations/0001_01_02_000020_create_candidates_table.php +++ b/database/migrations/0001_01_02_000020_create_candidates_table.php @@ -29,7 +29,7 @@ public function up(): void $table->timestamps(); - $table->unique(['name', 'election_id']); + $table->unique(['name', 'election_id', 'party_id']); }); } };