From 45aaa35a2759e0aa3f3c5bac367f12daadc40a9c Mon Sep 17 00:00:00 2001 From: tuutti Date: Tue, 1 Dec 2020 14:32:45 +0200 Subject: [PATCH 01/14] UHF-89: Initial commit --- .../migrate_plus.migration.tpr_unit.yml | 98 +---------- .../migrate_plus.migration.tpr_unit_en.yml | 99 ------------ .../migrate_plus.migration.tpr_unit_sv.yml | 99 ------------ ...age.content_settings.tpr_unit.tpr_unit.yml | 16 ++ src/Plugin/migrate/destination/Tpr.php | 14 ++ src/Plugin/migrate/destination/Unit.php | 110 +++++++++++++ src/Plugin/migrate/source/Tpr.php | 153 ++++++++++++++++++ 7 files changed, 296 insertions(+), 293 deletions(-) delete mode 100644 config/install/migrate_plus.migration.tpr_unit_en.yml delete mode 100644 config/install/migrate_plus.migration.tpr_unit_sv.yml create mode 100644 config/optional/language.content_settings.tpr_unit.tpr_unit.yml create mode 100644 src/Plugin/migrate/destination/Tpr.php create mode 100644 src/Plugin/migrate/destination/Unit.php create mode 100644 src/Plugin/migrate/source/Tpr.php diff --git a/config/install/migrate_plus.migration.tpr_unit.yml b/config/install/migrate_plus.migration.tpr_unit.yml index 3ac83fc8..81f439db 100644 --- a/config/install/migrate_plus.migration.tpr_unit.yml +++ b/config/install/migrate_plus.migration.tpr_unit.yml @@ -12,72 +12,8 @@ migration_tags: migration_group: tpr label: 'TPR unit' source: - plugin: url - urls: 'https://www.hel.fi/palvelukarttaws/rest/v4/unit/' - data_fetcher_plugin: http - data_parser_plugin: json - item_selector: 0 - fields: - - - name: id - selector: /id - - - name: name - selector: /name_fi - - - name: call_charge_info - selector: /call_charge_info_fi - - - name: latitude - selector: /latitude - - - name: longitude - selector: /longitude - - - name: phone - selector: /phone - - - name: email - selector: /email - - - name: short_desc - selector: /short_desc_fi - - - name: desc - selector: /desc_fi - - - name: street_address - selector: /street_address_fi - - - name: address_city - selector: /address_city_fi - - - name: address_postal_code - selector: /address_zip - - - name: accessibility_email - selector: /accessibility_email - - - name: accessibility_phone - selector: /accessibility_phone - - - name: accessibility_www - selector: /accessibility_www - - - name: address_postal - selector: /address_postal_full_fi - - - name: www - selector: /www_fi - - - name: streetview_entrance_url - selector: /streetview_entrance_url - - - name: created - selector: /created_time - - - name: changed - selector: /modified_time + plugin: tpr + url: 'https://www.hel.fi/palvelukarttaws/rest/v4/unit/?search=Pys%C3%A4k%C3%B6intilippuautomaatti' ids: id: type: string @@ -88,34 +24,6 @@ process: source: name start: 0 length: 255 - langcode: - plugin: default_value - default_value: fi - address/postal_code: address_postal_code - address/locality: address_city - address/address_line1: street_address - address/country_code: - plugin: default_value - default_value: FI - description/value: desc - description/summary: short_desc - description/format: - plugin: default_value - default_value: filtered_html - latitude: latitude - longitude: longitude - accessibility_email: accessibility_email - accessibility_phone: accessibility_phone - accessibility_www: accessibility_www - address_postal: address_postal - www/uri: www - streetview_entrance_url/uri: streetview_entrance_url - phone: - plugin: explode - source: phone - delimiter: ',' - call_charge_info: call_charge_info - email: email changed: - plugin: format_date @@ -135,5 +43,5 @@ process: from_timezone: 'Europe/Helsinki' to_timezone: 'UTC' destination: - plugin: 'entity:tpr_unit' + plugin: 'tpr_unit' migration_dependencies: [] diff --git a/config/install/migrate_plus.migration.tpr_unit_en.yml b/config/install/migrate_plus.migration.tpr_unit_en.yml deleted file mode 100644 index 52feb701..00000000 --- a/config/install/migrate_plus.migration.tpr_unit_en.yml +++ /dev/null @@ -1,99 +0,0 @@ -langcode: en -status: true -dependencies: - enforced: - module: - - helfi_tpr -id: tpr_unit_en -class: null -field_plugin_method: null -cck_plugin_method: null -migration_tags: - - tpr -migration_group: tpr -label: 'TPR unit (en)' -source: - plugin: url - urls: 'https://www.hel.fi/palvelukarttaws/rest/v4/unit/?' - data_fetcher_plugin: http - data_parser_plugin: json - item_selector: 0 - fields: - - - name: id - selector: /id - - - name: name - selector: /name_en - - - name: call_charge_info - selector: /call_charge_info_en - - - name: short_desc - selector: /short_desc_en - - - name: desc - selector: /desc_en - - - name: street_address - selector: /street_address_en - - - name: address_city - selector: /address_city_en - - - name: address_postal_code - selector: /address_zip - - - name: address_postal - selector: /address_postal_full_en - - - name: www - selector: /www_en - - - name: changed - selector: /modified_time - ids: - id: - type: string -process: - id: id - name: - plugin: substr - source: name - start: 0 - length: 255 - content_translation_source: - plugin: default_value - default_value: fi - langcode: - plugin: default_value - default_value: en - address/postal_code: address_postal_code - address/locality: address_city - address/address_line1: street_address - address/country_code: - plugin: default_value - default_value: FI - description/value: desc - description/summary: short_desc - description/format: - plugin: default_value - default_value: filtered_html - call_charge_info: call_charge_info - changed: - - - plugin: format_date - source: changed - from_format: 'Y-m-d\TH:i:s' - to_format: 'U' - from_timezone: 'Europe/Helsinki' - to_timezone: 'UTC' - - - plugin: entity_has_changed - entity_type: tpr_unit -destination: - plugin: 'entity:tpr_unit' - translations: true -migration_dependencies: - required: - - tpr_unit diff --git a/config/install/migrate_plus.migration.tpr_unit_sv.yml b/config/install/migrate_plus.migration.tpr_unit_sv.yml deleted file mode 100644 index 81de72b3..00000000 --- a/config/install/migrate_plus.migration.tpr_unit_sv.yml +++ /dev/null @@ -1,99 +0,0 @@ -langcode: en -status: true -dependencies: - enforced: - module: - - helfi_tpr -id: tpr_unit_sv -class: null -field_plugin_method: null -cck_plugin_method: null -migration_tags: - - tpr -migration_group: tpr -label: 'TPR unit (sv)' -source: - plugin: url - urls: 'https://www.hel.fi/palvelukarttaws/rest/v4/unit/' - data_fetcher_plugin: http - data_parser_plugin: json - item_selector: 0 - fields: - - - name: id - selector: /id - - - name: name - selector: /name_sv - - - name: call_charge_info - selector: /call_charge_info_sv - - - name: short_desc - selector: /short_desc_sv - - - name: desc - selector: /desc_sv - - - name: street_address - selector: /street_address_sv - - - name: address_city - selector: /address_city_sv - - - name: address_postal_code - selector: /address_zip - - - name: address_postal - selector: /address_postal_full_sv - - - name: www - selector: /www_sv - - - name: changed - selector: /modified_time - ids: - id: - type: string -process: - id: id - name: - plugin: substr - source: name - start: 0 - length: 255 - content_translation_source: - plugin: default_value - default_value: fi - langcode: - plugin: default_value - default_value: sv - address/postal_code: address_postal_code - address/locality: address_city - address/address_line1: street_address - address/country_code: - plugin: default_value - default_value: FI - description/value: desc - description/summary: short_desc - description/format: - plugin: default_value - default_value: filtered_html - call_charge_info: call_charge_info - changed: - - - plugin: format_date - source: changed - from_format: 'Y-m-d\TH:i:s' - to_format: 'U' - from_timezone: 'Europe/Helsinki' - to_timezone: 'UTC' - - - plugin: entity_has_changed - entity_type: tpr_unit -destination: - plugin: 'entity:tpr_unit' - translations: true -migration_dependencies: - required: - - tpr_unit diff --git a/config/optional/language.content_settings.tpr_unit.tpr_unit.yml b/config/optional/language.content_settings.tpr_unit.tpr_unit.yml new file mode 100644 index 00000000..9d9b55fc --- /dev/null +++ b/config/optional/language.content_settings.tpr_unit.tpr_unit.yml @@ -0,0 +1,16 @@ +langcode: en +status: true +dependencies: + module: + - content_translation + - helfi_tpr +third_party_settings: + content_translation: + enabled: true + bundle_settings: + untranslatable_fields_hide: '0' +id: tpr_unit.tpr_unit +target_entity_type_id: tpr_unit +target_bundle: tpr_unit +default_langcode: site_default +language_alterable: true diff --git a/src/Plugin/migrate/destination/Tpr.php b/src/Plugin/migrate/destination/Tpr.php new file mode 100644 index 00000000..66cf82a6 --- /dev/null +++ b/src/Plugin/migrate/destination/Tpr.php @@ -0,0 +1,14 @@ +languageManager = $container->get('language_manager'); + + return $instance; + } + + /** + * {@inheritdoc} + */ + protected static function getEntityTypeId($plugin_id) { + return 'tpr_unit'; + } + + /** + * {@inheritdoc} + */ + protected function getEntity(Row $row, array $old_destination_id_values) { + $default_language = $this->languageManager->getDefaultLanguage(); + $row = $this->populateRowFields($default_language, $row); + /** @var \Drupal\helfi_tpr\Entity\Unit $entity */ + $entity = parent::getEntity($row, $old_destination_id_values); + + $languages = $this->languageManager->getLanguages(); + unset($languages[$default_language->getId()]); + + foreach ($this->languageManager->getLanguages() as $langcode => $language) { + $languageRow = $this->populateRowFields($language, $row); + + if ($entity->hasTranslation($langcode)) { + // Update existing translation. + $translation = $entity->getTranslation($langcode); + $this->updateEntity($translation, $languageRow); + } + else { + // Stubs might need some required fields filled in. + if ($languageRow->isStub()) { + $this->processStubRow($languageRow); + } + $translation = $entity->addTranslation($langcode, $languageRow->getDestination()); + $translation->enforceIsNew(); + } + } + return $entity; + } + + protected function populateRowFields(LanguageInterface $language, Row $row) : Row { + $langcode = $language->getId(); + + if (!$row->get('langcode')) { + $row->setDestinationProperty('langcode', $langcode); + } + + foreach (['name'] as $name) { + $field = sprintf('%s_%s', $name, $langcode); + + // Attempt to read source property in current language and fallback to + // finnish. + $value = $row->hasSourceProperty($field) ? $row->getSourceProperty($field) : $row->getSourceProperty(sprintf('%s_fi', $name)); + $row->setDestinationProperty($name, $value); + } + + return $row; + } + + /** + * {@inheritdoc} + */ + public function getIds() { + return ['id' => ['type' => 'string']]; + } + +} diff --git a/src/Plugin/migrate/source/Tpr.php b/src/Plugin/migrate/source/Tpr.php new file mode 100644 index 00000000..7b70deeb --- /dev/null +++ b/src/Plugin/migrate/source/Tpr.php @@ -0,0 +1,153 @@ + ['type' => 'string']]; + } + + /** + * {@inheritdoc} + */ + public function fields() { + return []; + } + + /** + * {@inheritdoc} + */ + public function next() { + parent::next(); + + // Check if the current row has changes and increment ignoredRows variable + // to allow us to stop migrate early if we have no changes. + if ($this->isPartialMigrate() && $this->currentRow && !$this->currentRow->changed()) { + $this->ignoredRows++; + } + } + + /** + * Sends a HTTP request and returns response data as array. + * + * @param string $url + * The url. + * + * @return array + * The JSON returned by Ahjo service. + */ + protected function getContent(string $url) : array { + try { + $content = (string) $this->httpClient->request('GET', $url)->getBody(); + + $json = \GuzzleHttp\json_decode($content, TRUE); + + $dates = []; + // Sort data by modified_time. + foreach ($json as $key => $item) { + $dates[$key] = DateTimePlus::createFromFormat('Y-m-d\TH:i:s', $item['modified_time'])->format('U'); + } + array_multisort($dates, SORT_DESC, $json); + + return $json; + } + catch (GuzzleException $e) { + } + return []; + } + + /** + * {@inheritdoc} + */ + protected function initializeIterator() { + $content = $this->getContent($this->configuration['url']); + + foreach ($content as $object) { + // Skip entire migration once we've reached the number of maximum + // ignored (not changed) rows. + // @see static::NUM_IGNORED_ROWS_BEFORE_STOPPING. + if ($this->isPartialMigrate() && ($this->ignoredRows >= static::NUM_IGNORED_ROWS_BEFORE_STOPPING)) { + break; + } + yield $object; + } + } + + /** + * {@inheritdoc} + */ + public static function create( + ContainerInterface $container, + array $configuration, + $plugin_id, + $plugin_definition, + MigrationInterface $migration = NULL + ) { + $instance = new static($configuration, $plugin_id, $plugin_definition, $migration); + $instance->httpClient = $container->get('http_client'); + + if (!isset($configuration['url'])) { + throw new \InvalidArgumentException('The "url" configuration missing.'); + } + return $instance; + } + +} From 9a54befaec29868b0da064ad192fb5f56ff761df Mon Sep 17 00:00:00 2001 From: tuutti Date: Tue, 1 Dec 2020 14:58:03 +0200 Subject: [PATCH 02/14] UHF-89: Abstract translatable fields to make it more re-usable --- src/Plugin/migrate/destination/Tpr.php | 79 +++++++++++++++++++++++++ src/Plugin/migrate/destination/Unit.php | 63 ++------------------ 2 files changed, 85 insertions(+), 57 deletions(-) diff --git a/src/Plugin/migrate/destination/Tpr.php b/src/Plugin/migrate/destination/Tpr.php index 66cf82a6..7a0b6c16 100644 --- a/src/Plugin/migrate/destination/Tpr.php +++ b/src/Plugin/migrate/destination/Tpr.php @@ -4,11 +4,90 @@ namespace Drupal\helfi_tpr\Plugin\migrate\destination; +use Drupal\Core\Language\LanguageInterface; use Drupal\migrate\Plugin\migrate\destination\EntityContentBase; +use Drupal\migrate\Row; /** * Provides a destination plugin for Tpr entities. */ abstract class Tpr extends EntityContentBase { + /** + * {@inheritdoc} + */ + protected function getEntity(Row $row, array $old_destination_id_values) { + $default_language = $this->languageManager->getDefaultLanguage(); + $row = $this->populateRowFields($default_language, $row); + /** @var \Drupal\helfi_tpr\Entity\Unit $entity */ + $entity = parent::getEntity($row, $old_destination_id_values); + + $languages = $this->languageManager->getLanguages(); + unset($languages[$default_language->getId()]); + + foreach ($this->languageManager->getLanguages() as $langcode => $language) { + $languageRow = $this->populateRowFields($language, $row); + + if ($entity->hasTranslation($langcode)) { + // Update existing translation. + $translation = $entity->getTranslation($langcode); + $this->updateEntity($translation, $languageRow); + } + else { + // Stubs might need some required fields filled in. + if ($languageRow->isStub()) { + $this->processStubRow($languageRow); + } + $translation = $entity->addTranslation($langcode, $languageRow->getDestination()); + $translation->enforceIsNew(); + } + } + return $entity; + } + + /** + * Gets the translatable source fields. + * + * @return string[] + * An array of source fields. + */ + abstract protected function getTranslatableFields() : array; + + /** + * Populates the row object values. + * + * @param \Drupal\Core\Language\LanguageInterface $language + * The language. + * @param \Drupal\migrate\Row $row + * The row. + * + * @return \Drupal\migrate\Row + * The row. + */ + protected function populateRowFields(LanguageInterface $language, Row $row) : Row { + $langcode = $language->getId(); + + if (!$row->get('langcode')) { + $row->setDestinationProperty('langcode', $langcode); + } + + foreach ($this->getTranslatableFields() as $name) { + $field = sprintf('%s_%s', $name, $langcode); + + // Attempt to read source property in current language and fallback to + // finnish. + $value = $row->hasSourceProperty($field) ? $row->getSourceProperty($field) : $row->getSourceProperty(sprintf('%s_fi', $name)); + $row->setDestinationProperty($name, $value); + } + + return $row; + } + + /** + * {@inheritdoc} + */ + public function getIds() { + return ['id' => ['type' => 'string']]; + } + } diff --git a/src/Plugin/migrate/destination/Unit.php b/src/Plugin/migrate/destination/Unit.php index 7249c52e..240402db 100644 --- a/src/Plugin/migrate/destination/Unit.php +++ b/src/Plugin/migrate/destination/Unit.php @@ -4,10 +4,8 @@ namespace Drupal\helfi_tpr\Plugin\migrate\destination; -use Drupal\Core\Language\LanguageInterface; use Drupal\Core\Language\LanguageManagerInterface; use Drupal\migrate\Plugin\MigrationInterface; -use Drupal\migrate\Row; use Symfony\Component\DependencyInjection\ContainerInterface; /** @@ -45,66 +43,17 @@ public static function create( /** * {@inheritdoc} */ - protected static function getEntityTypeId($plugin_id) { - return 'tpr_unit'; - } - - /** - * {@inheritdoc} - */ - protected function getEntity(Row $row, array $old_destination_id_values) { - $default_language = $this->languageManager->getDefaultLanguage(); - $row = $this->populateRowFields($default_language, $row); - /** @var \Drupal\helfi_tpr\Entity\Unit $entity */ - $entity = parent::getEntity($row, $old_destination_id_values); - - $languages = $this->languageManager->getLanguages(); - unset($languages[$default_language->getId()]); - - foreach ($this->languageManager->getLanguages() as $langcode => $language) { - $languageRow = $this->populateRowFields($language, $row); - - if ($entity->hasTranslation($langcode)) { - // Update existing translation. - $translation = $entity->getTranslation($langcode); - $this->updateEntity($translation, $languageRow); - } - else { - // Stubs might need some required fields filled in. - if ($languageRow->isStub()) { - $this->processStubRow($languageRow); - } - $translation = $entity->addTranslation($langcode, $languageRow->getDestination()); - $translation->enforceIsNew(); - } - } - return $entity; - } - - protected function populateRowFields(LanguageInterface $language, Row $row) : Row { - $langcode = $language->getId(); - - if (!$row->get('langcode')) { - $row->setDestinationProperty('langcode', $langcode); - } - - foreach (['name'] as $name) { - $field = sprintf('%s_%s', $name, $langcode); - - // Attempt to read source property in current language and fallback to - // finnish. - $value = $row->hasSourceProperty($field) ? $row->getSourceProperty($field) : $row->getSourceProperty(sprintf('%s_fi', $name)); - $row->setDestinationProperty($name, $value); - } - - return $row; + protected function getTranslatableFields(): array { + return [ + 'name', + ]; } /** * {@inheritdoc} */ - public function getIds() { - return ['id' => ['type' => 'string']]; + protected static function getEntityTypeId($plugin_id) { + return 'tpr_unit'; } } From 63a2caa5b0154b4115056940120d26229f25a2ec Mon Sep 17 00:00:00 2001 From: tuutti Date: Wed, 2 Dec 2020 12:18:05 +0200 Subject: [PATCH 03/14] UHF-89: Fetch extra fields from individual unit pages --- src/Plugin/migrate/destination/Tpr.php | 6 ++-- src/Plugin/migrate/source/Tpr.php | 44 +++++++++++++++++++------- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/Plugin/migrate/destination/Tpr.php b/src/Plugin/migrate/destination/Tpr.php index 7a0b6c16..6b51e356 100644 --- a/src/Plugin/migrate/destination/Tpr.php +++ b/src/Plugin/migrate/destination/Tpr.php @@ -18,7 +18,7 @@ abstract class Tpr extends EntityContentBase { */ protected function getEntity(Row $row, array $old_destination_id_values) { $default_language = $this->languageManager->getDefaultLanguage(); - $row = $this->populateRowFields($default_language, $row); + $row = $this->populateFieldTranslations($default_language, $row); /** @var \Drupal\helfi_tpr\Entity\Unit $entity */ $entity = parent::getEntity($row, $old_destination_id_values); @@ -26,7 +26,7 @@ protected function getEntity(Row $row, array $old_destination_id_values) { unset($languages[$default_language->getId()]); foreach ($this->languageManager->getLanguages() as $langcode => $language) { - $languageRow = $this->populateRowFields($language, $row); + $languageRow = $this->populateFieldTranslations($language, $row); if ($entity->hasTranslation($langcode)) { // Update existing translation. @@ -64,7 +64,7 @@ abstract protected function getTranslatableFields() : array; * @return \Drupal\migrate\Row * The row. */ - protected function populateRowFields(LanguageInterface $language, Row $row) : Row { + protected function populateFieldTranslations(LanguageInterface $language, Row $row) : Row { $langcode = $language->getId(); if (!$row->get('langcode')) { diff --git a/src/Plugin/migrate/source/Tpr.php b/src/Plugin/migrate/source/Tpr.php index 7b70deeb..6ed066f2 100644 --- a/src/Plugin/migrate/source/Tpr.php +++ b/src/Plugin/migrate/source/Tpr.php @@ -5,6 +5,7 @@ namespace Drupal\helfi_tpr\Plugin\migrate\source; use Drupal\Component\Datetime\DateTimePlus; +use Drupal\Component\Utility\UrlHelper; use Drupal\Core\Plugin\ContainerFactoryPluginInterface; use Drupal\helfi_api_base\MigrateTrait; use Drupal\migrate\Plugin\migrate\source\SourcePluginBase; @@ -97,29 +98,45 @@ public function next() { protected function getContent(string $url) : array { try { $content = (string) $this->httpClient->request('GET', $url)->getBody(); - - $json = \GuzzleHttp\json_decode($content, TRUE); - - $dates = []; - // Sort data by modified_time. - foreach ($json as $key => $item) { - $dates[$key] = DateTimePlus::createFromFormat('Y-m-d\TH:i:s', $item['modified_time'])->format('U'); - } - array_multisort($dates, SORT_DESC, $json); - - return $json; + return \GuzzleHttp\json_decode($content, TRUE); } catch (GuzzleException $e) { } return []; } + /** + * Builds a canonical url to individual entity. + * + * @param int $id + * The entity ID. + * + * @return string + * The url to canonical page of given entity. + */ + private function buildCanonicalUrl(int $id) : string { + $urlParts = UrlHelper::parse($this->configuration['url']); + + return vsprintf('%s/%s/?%s', [ + rtrim($urlParts['path'], '/'), + $id, + UrlHelper::buildQuery($urlParts['query']), + ]); + } + /** * {@inheritdoc} */ protected function initializeIterator() { $content = $this->getContent($this->configuration['url']); + $dates = []; + // Sort data by modified_time. + foreach ($content as $key => $item) { + $dates[$key] = DateTimePlus::createFromFormat('Y-m-d\TH:i:s', $item['modified_time'])->format('U'); + } + array_multisort($dates, SORT_DESC, $content); + foreach ($content as $object) { // Skip entire migration once we've reached the number of maximum // ignored (not changed) rows. @@ -127,6 +144,8 @@ protected function initializeIterator() { if ($this->isPartialMigrate() && ($this->ignoredRows >= static::NUM_IGNORED_ROWS_BEFORE_STOPPING)) { break; } + $object += $this->getContent($this->buildCanonicalUrl($object['id'])); + yield $object; } } @@ -141,7 +160,8 @@ public static function create( $plugin_definition, MigrationInterface $migration = NULL ) { - $instance = new static($configuration, $plugin_id, $plugin_definition, $migration); + $instance = new static($configuration, $plugin_id, $plugin_definition, + $migration); $instance->httpClient = $container->get('http_client'); if (!isset($configuration['url'])) { From 9f25dfd4c58c83188b52f428de00d12ec4f50c2b Mon Sep 17 00:00:00 2001 From: tuutti Date: Wed, 2 Dec 2020 12:25:39 +0200 Subject: [PATCH 04/14] UHF-89: Moved object initialization to correct file --- src/Plugin/migrate/destination/Tpr.php | 26 ++++++++++++++++++++++++ src/Plugin/migrate/destination/Unit.php | 27 ------------------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/src/Plugin/migrate/destination/Tpr.php b/src/Plugin/migrate/destination/Tpr.php index 6b51e356..644220da 100644 --- a/src/Plugin/migrate/destination/Tpr.php +++ b/src/Plugin/migrate/destination/Tpr.php @@ -5,14 +5,40 @@ namespace Drupal\helfi_tpr\Plugin\migrate\destination; use Drupal\Core\Language\LanguageInterface; +use Drupal\Core\Language\LanguageManagerInterface; use Drupal\migrate\Plugin\migrate\destination\EntityContentBase; +use Drupal\migrate\Plugin\MigrationInterface; use Drupal\migrate\Row; +use Symfony\Component\DependencyInjection\ContainerInterface; /** * Provides a destination plugin for Tpr entities. */ abstract class Tpr extends EntityContentBase { + /** + * The language manager service. + * + * @var \Drupal\Core\Language\LanguageManagerInterface + */ + protected LanguageManagerInterface $languageManager; + + /** + * {@inheritdoc} + */ + public static function create( + ContainerInterface $container, + array $configuration, + $plugin_id, + $plugin_definition, + MigrationInterface $migration = NULL + ) { + $instance = parent::create($container, $configuration, $plugin_id, $plugin_definition, $migration); + $instance->languageManager = $container->get('language_manager'); + + return $instance; + } + /** * {@inheritdoc} */ diff --git a/src/Plugin/migrate/destination/Unit.php b/src/Plugin/migrate/destination/Unit.php index 240402db..e9ec9edb 100644 --- a/src/Plugin/migrate/destination/Unit.php +++ b/src/Plugin/migrate/destination/Unit.php @@ -4,10 +4,6 @@ namespace Drupal\helfi_tpr\Plugin\migrate\destination; -use Drupal\Core\Language\LanguageManagerInterface; -use Drupal\migrate\Plugin\MigrationInterface; -use Symfony\Component\DependencyInjection\ContainerInterface; - /** * Provides a destination plugin for Tpr entities. * @@ -17,29 +13,6 @@ */ final class Unit extends Tpr { - /** - * The language manager service. - * - * @var \Drupal\Core\Language\LanguageManagerInterface - */ - protected LanguageManagerInterface $languageManager; - - /** - * {@inheritdoc} - */ - public static function create( - ContainerInterface $container, - array $configuration, - $plugin_id, - $plugin_definition, - MigrationInterface $migration = NULL - ) { - $instance = parent::create($container, $configuration, $plugin_id, $plugin_definition, $migration); - $instance->languageManager = $container->get('language_manager'); - - return $instance; - } - /** * {@inheritdoc} */ From d261b3c14c1a578905551c8e1f340fd551701d34 Mon Sep 17 00:00:00 2001 From: tuutti Date: Wed, 2 Dec 2020 15:25:44 +0200 Subject: [PATCH 05/14] UHF-89: Initial working unit migration --- .../migrate_plus.migration.tpr_unit.yml | 49 ++++++++++++------- .../migrate_plus.migration_group.tpr.yml | 9 ++++ src/Entity/Unit.php | 2 +- src/Plugin/migrate/destination/Tpr.php | 23 +++++++-- src/Plugin/migrate/destination/Unit.php | 9 +++- src/Plugin/migrate/source/Tpr.php | 15 ++++++ 6 files changed, 82 insertions(+), 25 deletions(-) create mode 100644 config/install/migrate_plus.migration_group.tpr.yml diff --git a/config/install/migrate_plus.migration.tpr_unit.yml b/config/install/migrate_plus.migration.tpr_unit.yml index 81f439db..84815eff 100644 --- a/config/install/migrate_plus.migration.tpr_unit.yml +++ b/config/install/migrate_plus.migration.tpr_unit.yml @@ -1,8 +1,11 @@ +langcode: en status: true dependencies: enforced: module: - helfi_tpr +_core: + default_config_hash: utEV05AAC3Rkgn-SLMNDjfNlAGdILfP-w3mJ0HF3MNQ id: tpr_unit class: null field_plugin_method: null @@ -13,35 +16,43 @@ migration_group: tpr label: 'TPR unit' source: plugin: tpr - url: 'https://www.hel.fi/palvelukarttaws/rest/v4/unit/?search=Pys%C3%A4k%C3%B6intilippuautomaatti' + track_changes: true + url: 'https://www.hel.fi/palvelukarttaws/rest/v4/unit/' ids: id: type: string process: id: id - name: - plugin: substr - source: name - start: 0 - length: 255 + latitude: latitude + longitude: longitude + streetview_entrance_url/uri: streetview_entrance_url + description/format: + plugin: default_value + default_value: filtered_html + address/postal_code: address_zip + address/country_code: + plugin: default_value + default_value: FI + phone: + plugin: explode + source: phone + delimiter: ',' + strict: false changed: - plugin: format_date - source: changed + source: modified_time from_format: 'Y-m-d\TH:i:s' - to_format: 'U' - from_timezone: 'Europe/Helsinki' - to_timezone: 'UTC' - - - plugin: entity_has_changed - entity_type: tpr_unit + to_format: U + from_timezone: Europe/Helsinki + to_timezone: UTC created: plugin: format_date - source: created + source: created_time from_format: 'Y-m-d\TH:i:s' - to_format: 'U' - from_timezone: 'Europe/Helsinki' - to_timezone: 'UTC' + to_format: U + from_timezone: Europe/Helsinki + to_timezone: UTC destination: - plugin: 'tpr_unit' -migration_dependencies: [] + plugin: tpr_unit +migration_dependencies: { } diff --git a/config/install/migrate_plus.migration_group.tpr.yml b/config/install/migrate_plus.migration_group.tpr.yml new file mode 100644 index 00000000..c06b173c --- /dev/null +++ b/config/install/migrate_plus.migration_group.tpr.yml @@ -0,0 +1,9 @@ +langcode: en +status: true +dependencies: { } +id: tpr +label: TPR +description: '' +source_type: null +module: helfi_tpr +shared_configuration: null diff --git a/src/Entity/Unit.php b/src/Entity/Unit.php index 577c1f79..df07416d 100644 --- a/src/Entity/Unit.php +++ b/src/Entity/Unit.php @@ -72,7 +72,7 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) { ->setTranslatable(FALSE); $fields['longitude'] = static::createStringField('Longitude') ->setTranslatable(FALSE); - $fields['phone'] = static::createStringField('Phone', -1) + $fields['phone'] = static::createStringField('Phone', BaseFieldDefinition::CARDINALITY_UNLIMITED) ->setTranslatable(FALSE); $fields['call_charge_info'] = static::createStringField('Call charge info'); $fields['email'] = static::createStringField('Email') diff --git a/src/Plugin/migrate/destination/Tpr.php b/src/Plugin/migrate/destination/Tpr.php index 644220da..b4bc843c 100644 --- a/src/Plugin/migrate/destination/Tpr.php +++ b/src/Plugin/migrate/destination/Tpr.php @@ -74,6 +74,17 @@ protected function getEntity(Row $row, array $old_destination_id_values) { /** * Gets the translatable source fields. * + * Defined as remote field name => local field name: + * + * @code + * [ + * 'name => 'field_name', + * 'www' => 'field_url', + * ] + * @endcode + * Language code will be appended to remote field automatically. For + * example the field `name` will become name_fi, name_en etc. + * * @return string[] * An array of source fields. */ @@ -97,13 +108,17 @@ protected function populateFieldTranslations(LanguageInterface $language, Row $r $row->setDestinationProperty('langcode', $langcode); } - foreach ($this->getTranslatableFields() as $name) { - $field = sprintf('%s_%s', $name, $langcode); + foreach ($this->getTranslatableFields() as $remote => $local) { + $field = sprintf('%s_%s', $remote, $langcode); // Attempt to read source property in current language and fallback to // finnish. - $value = $row->hasSourceProperty($field) ? $row->getSourceProperty($field) : $row->getSourceProperty(sprintf('%s_fi', $name)); - $row->setDestinationProperty($name, $value); + $value = $row->hasSourceProperty($field) ? $row->getSourceProperty($field) : $row->getSourceProperty(sprintf('%s_fi', $remote)); + + if (!$value) { + continue; + } + $row->setDestinationProperty($local, $value); } return $row; diff --git a/src/Plugin/migrate/destination/Unit.php b/src/Plugin/migrate/destination/Unit.php index e9ec9edb..34a64ce3 100644 --- a/src/Plugin/migrate/destination/Unit.php +++ b/src/Plugin/migrate/destination/Unit.php @@ -18,7 +18,14 @@ final class Unit extends Tpr { */ protected function getTranslatableFields(): array { return [ - 'name', + 'name' => 'name', + 'call_charge_info' => 'call_charge_info', + 'www' => 'www/uri', + 'address_postal_full' => 'address_postal', + 'street_address' => 'address/address_line1', + 'address_city' => 'address/locality', + 'desc' => 'description/value', + 'short_desc' => 'description/summary', ]; } diff --git a/src/Plugin/migrate/source/Tpr.php b/src/Plugin/migrate/source/Tpr.php index 6ed066f2..e22abf79 100644 --- a/src/Plugin/migrate/source/Tpr.php +++ b/src/Plugin/migrate/source/Tpr.php @@ -52,6 +52,13 @@ class Tpr extends SourcePluginBase implements ContainerFactoryPluginInterface { */ protected int $ignoredRows = 0; + /** + * The total count. + * + * @var int + */ + protected int $count = 0; + /** * {@inheritdoc} */ @@ -66,6 +73,13 @@ public function getIds() { return ['id' => ['type' => 'string']]; } + /** + * {@inheritdoc} + */ + public function count($refresh = FALSE) { + return $this->count; + } + /** * {@inheritdoc} */ @@ -129,6 +143,7 @@ private function buildCanonicalUrl(int $id) : string { */ protected function initializeIterator() { $content = $this->getContent($this->configuration['url']); + $this->count = count($content); $dates = []; // Sort data by modified_time. From 667627c10061ebaf8eb40749df420aa68b79ae0d Mon Sep 17 00:00:00 2001 From: tuutti Date: Thu, 3 Dec 2020 13:44:43 +0200 Subject: [PATCH 06/14] UHF-89: Removed default hash --- config/install/migrate_plus.migration.tpr_unit.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/install/migrate_plus.migration.tpr_unit.yml b/config/install/migrate_plus.migration.tpr_unit.yml index 84815eff..1e6aaaa4 100644 --- a/config/install/migrate_plus.migration.tpr_unit.yml +++ b/config/install/migrate_plus.migration.tpr_unit.yml @@ -4,8 +4,6 @@ dependencies: enforced: module: - helfi_tpr -_core: - default_config_hash: utEV05AAC3Rkgn-SLMNDjfNlAGdILfP-w3mJ0HF3MNQ id: tpr_unit class: null field_plugin_method: null From 072ef5868e99bb91a7d740476ce5ca5322788ad3 Mon Sep 17 00:00:00 2001 From: tuutti Date: Thu, 3 Dec 2020 14:47:54 +0200 Subject: [PATCH 07/14] UHF-89: Made language setting non-optional, fixed tests --- ...age.content_settings.tpr_unit.tpr_unit.yml | 0 tests/src/Kernel/UnitMigrationTest.php | 49 ++++++------------- 2 files changed, 16 insertions(+), 33 deletions(-) rename config/{optional => install}/language.content_settings.tpr_unit.tpr_unit.yml (100%) diff --git a/config/optional/language.content_settings.tpr_unit.tpr_unit.yml b/config/install/language.content_settings.tpr_unit.tpr_unit.yml similarity index 100% rename from config/optional/language.content_settings.tpr_unit.tpr_unit.yml rename to config/install/language.content_settings.tpr_unit.tpr_unit.yml diff --git a/tests/src/Kernel/UnitMigrationTest.php b/tests/src/Kernel/UnitMigrationTest.php index 4490c386..ba671fc9 100644 --- a/tests/src/Kernel/UnitMigrationTest.php +++ b/tests/src/Kernel/UnitMigrationTest.php @@ -6,6 +6,7 @@ use Drupal\helfi_tpr\Entity\Unit; use Drupal\Tests\helfi_api_base\Kernel\MigrationTestBase; +use GuzzleHttp\Psr7\Response; /** * Tests unit migration. @@ -36,45 +37,27 @@ public function setUp() : void { /** * Test default migrations. - * - * @dataProvider migrationsDataProvider */ - public function testMigration(string $migrate, int $expectedCount, string $langcode) : void { - - foreach (['tpr_unit', $migrate] as $item) { - // Override default url with local copy of unit data. - $config = $this->config('migrate_plus.migration.' . $item); - - $overrides = [ - 'urls' => $this->getFixturePath('helfi_tpr', 'unit.json'), - 'data_fetcher_plugin' => 'file', - ]; - $config->set('source', $overrides + $config->get('source')) - ->save(); - - $this->flushPluginCache(); + public function testMigration() : void { + $units = $this->getFixture('helfi_tpr', 'unit.json'); + $responses = [ + new Response(200, [], $units), + ]; - $this->executeMigration($item); + foreach (json_decode($units, TRUE) as $id => $unit) { + $responses[] = new Response(200, [], json_encode($unit)); } + + $this->container->set('http_client', $this->createMockHttpClient($responses)); + $this->executeMigration('tpr_unit'); $entities = Unit::loadMultiple(); - $this->assertCount($expectedCount, $entities); + $this->assertCount(6, $entities); - foreach ($entities as $entity) { - $this->assertEqual($entity->getTranslation($langcode)->language()->getId(), $langcode); + foreach (['en', 'sv'] as $langcode) { + foreach ($entities as $entity) { + $this->assertEqual($entity->getTranslation($langcode)->language()->getId(), $langcode); + } } } - /** - * Gets the migration data. - * - * @return array[] - * The migration. - */ - public function migrationsDataProvider() : array { - return [ - ['tpr_unit_sv', 6, 'sv'], - ['tpr_unit_en', 6, 'en'], - ]; - } - } From 5330960d9011101eab278f7eadc00b41ae061dba Mon Sep 17 00:00:00 2001 From: tuutti Date: Thu, 3 Dec 2020 15:06:24 +0200 Subject: [PATCH 08/14] UHF-89: Missing dependencies --- helfi_tpr.info.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helfi_tpr.info.yml b/helfi_tpr.info.yml index e8ae7808..db340eb2 100644 --- a/helfi_tpr.info.yml +++ b/helfi_tpr.info.yml @@ -12,3 +12,5 @@ dependencies: - migrate - migrate_plus - migrate_tools + - content_translation + - language From afbadf5bda4344fd1e08329dc41e5eac177a7fe4 Mon Sep 17 00:00:00 2001 From: tuutti Date: Thu, 3 Dec 2020 15:06:42 +0200 Subject: [PATCH 09/14] UHF-89: Removed composer debug --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aee5a540..536ab870 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,6 @@ jobs: ln -s $GITHUB_WORKSPACE $HOME/drupal/public/modules/custom cd $HOME/drupal && echo "$(jq --indent 4 '.repositories |= [{"type": "path", "url": "'$GITHUB_WORKSPACE'"}] + .' composer.json)" > composer.json && composer require drupal/$MODULE_NAME - cat $HOME/drupal/composer.json drush en --root $HOME/drupal $MODULE_NAME -y - name: Run PHPCS From 3cad2b4df42b296bce69f21d0d55e1ad8b7ee2b9 Mon Sep 17 00:00:00 2001 From: tuutti Date: Thu, 3 Dec 2020 15:10:28 +0200 Subject: [PATCH 10/14] UHF-89: Added prophecy dependency --- composer.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/composer.json b/composer.json index be3fa19d..82c3d8c1 100644 --- a/composer.json +++ b/composer.json @@ -12,6 +12,7 @@ }, "require-dev": { "dealerdirect/phpcodesniffer-composer-installer": "^0.7.0", - "drupal/coder": "^8.3" + "drupal/coder": "^8.3", + "phpspec/prophecy-phpunit": "^2" } } From 3f8e39823e7d22855634fd1f55d47557a253e5aa Mon Sep 17 00:00:00 2001 From: tuutti Date: Thu, 3 Dec 2020 15:16:19 +0200 Subject: [PATCH 11/14] UHF-89: Require prophecy --- .github/workflows/ci.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 536ab870..2ba79544 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,14 +22,17 @@ jobs: with: fetch-depth: 1 + - name: Set variables + run: echo "DRUPAL_ROOT=$HOME/drupal" >> $GITHUB_ENV + - name: Build project run: | - composer create-project City-of-Helsinki/drupal-helfi-platform:dev-main $HOME/drupal --no-interaction --repository https://repository.drupal.hel.ninja/ - php -d sendmail_path=$(which true); drush --yes --root $HOME/drupal -v site-install minimal --db-url="$SIMPLETEST_DB" - ln -s $GITHUB_WORKSPACE $HOME/drupal/public/modules/custom - cd $HOME/drupal && - echo "$(jq --indent 4 '.repositories |= [{"type": "path", "url": "'$GITHUB_WORKSPACE'"}] + .' composer.json)" > composer.json && composer require drupal/$MODULE_NAME - drush en --root $HOME/drupal $MODULE_NAME -y + composer create-project city-of-helsinki/drupal-helfi-platform:dev-main $drupal_root --no-interaction --repository https://repository.drupal.hel.ninja/ + cd $drupal_root + php -d sendmail_path=$(which true); drush --yes -v site-install minimal --db-url="$SIMPLETEST_DB" + echo "$(jq --indent 4 '.repositories |= [{"type": "path", "url": "'$github_workspace'"}] + .' composer.json)" > composer.json + composer require drupal/$module_name "phpspec/prophecy-phpunit ^2" + drush en $module_name -y - name: Run PHPCS run: cd $HOME/drupal && vendor/bin/phpcs public/modules/custom/ --standard=Drupal From 89d047dac125701094159069ac85b3bf40a3a749 Mon Sep 17 00:00:00 2001 From: tuutti Date: Thu, 3 Dec 2020 15:19:06 +0200 Subject: [PATCH 12/14] UHF-89: Fixed test command --- .github/workflows/ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ba79544..20c503a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,10 +35,13 @@ jobs: drush en $module_name -y - name: Run PHPCS - run: cd $HOME/drupal && vendor/bin/phpcs public/modules/custom/ --standard=Drupal + run: | + cd $DRUPAL_ROOT + vendor/bin/phpcs public/modules/custom/ --standard=Drupal - name: Run PHPUnit tests run: | - drush runserver $SIMPLETEST_BASE_URL --root $HOME/drupal > /dev/null 2>&1 & + cd $DRUPAL_ROOT + drush runserver $SIMPLETEST_BASE_URL > /dev/null 2>&1 & chromedriver --port=4444 > /dev/null 2>&1 & - cd $HOME/drupal && vendor/bin/phpunit -c $HOME/drupal/phpunit.xml.dist --verbose --group $MODULE_NAME + vendor/bin/phpunit -c $DRUPAL_ROOT/phpunit.xml.dist --verbose --group $MODULE_NAME From a2e4e453eac842b96997088e15716a195f0a1a57 Mon Sep 17 00:00:00 2001 From: tuutti Date: Thu, 3 Dec 2020 15:19:47 +0200 Subject: [PATCH 13/14] UHF-89: Fixed variables --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20c503a7..553a45c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,12 +27,12 @@ jobs: - name: Build project run: | - composer create-project city-of-helsinki/drupal-helfi-platform:dev-main $drupal_root --no-interaction --repository https://repository.drupal.hel.ninja/ - cd $drupal_root + composer create-project city-of-helsinki/drupal-helfi-platform:dev-main $DRUPAL_ROOT --no-interaction --repository https://repository.drupal.hel.ninja/ + cd $DRUPAL_ROOT php -d sendmail_path=$(which true); drush --yes -v site-install minimal --db-url="$SIMPLETEST_DB" - echo "$(jq --indent 4 '.repositories |= [{"type": "path", "url": "'$github_workspace'"}] + .' composer.json)" > composer.json - composer require drupal/$module_name "phpspec/prophecy-phpunit ^2" - drush en $module_name -y + echo "$(jq --indent 4 '.repositories |= [{"type": "path", "url": "'$GITHUB_WORKSPACE'"}] + .' composer.json)" > composer.json + composer require drupal/$MODULE_NAME "phpspec/prophecy-phpunit ^2" + drush en $MODULE_NAME -y - name: Run PHPCS run: | From 5ba03d02d7ad81e37c0c492aac5131ee5968c00a Mon Sep 17 00:00:00 2001 From: tuutti Date: Thu, 3 Dec 2020 16:16:28 +0200 Subject: [PATCH 14/14] UHF-89: Disable deprecations for now --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 553a45c0..3a9d0581 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,7 @@ env: SIMPLETEST_BASE_URL: "http://127.0.0.1:8080" DRUPAL_CORE: 9.0.x MODULE_NAME: helfi_tpr + SYMFONY_DEPRECATIONS_HELPER: disabled jobs: tests: runs-on: ubuntu-latest