Merge pull request #5 from City-of-Helsinki/UHF-89_tpr

Refactor unit migration to fetch data from individual pages, better translation support

.github/workflows/ci.yml

@@ -5,6 +5,7 @@ env:
   SIMPLETEST_BASE_URL: "http://127.0.0.1:8080"
   DRUPAL_CORE: 9.0.x
   MODULE_NAME: helfi_tpr
+  SYMFONY_DEPRECATIONS_HELPER: disabled
 jobs:
   tests:
     runs-on: ubuntu-latest
@@ -22,21 +23,26 @@ jobs:
         with:
           fetch-depth: 1
 
+      - name: Set variables
+        run: echo "DRUPAL_ROOT=$HOME/drupal" >> $GITHUB_ENV
+
       - name: Build project
         run: |
-          composer create-project City-of-Helsinki/drupal-helfi-platform:dev-main $HOME/drupal --no-interaction --repository https://repository.drupal.hel.ninja/
-          php -d sendmail_path=$(which true); drush --yes --root $HOME/drupal -v site-install minimal --db-url="$SIMPLETEST_DB"
-          ln -s $GITHUB_WORKSPACE $HOME/drupal/public/modules/custom
-          cd $HOME/drupal &&
-          echo "$(jq --indent 4 '.repositories |= [{"type": "path", "url": "'$GITHUB_WORKSPACE'"}] + .' composer.json)" > composer.json && composer require drupal/$MODULE_NAME
-          cat $HOME/drupal/composer.json
-          drush en --root $HOME/drupal $MODULE_NAME -y
+          composer create-project city-of-helsinki/drupal-helfi-platform:dev-main $DRUPAL_ROOT --no-interaction --repository https://repository.drupal.hel.ninja/
+          cd $DRUPAL_ROOT
+          php -d sendmail_path=$(which true); drush --yes -v site-install minimal --db-url="$SIMPLETEST_DB"
+          echo "$(jq --indent 4 '.repositories |= [{"type": "path", "url": "'$GITHUB_WORKSPACE'"}] + .' composer.json)" > composer.json
+          composer require drupal/$MODULE_NAME "phpspec/prophecy-phpunit ^2"
+          drush en $MODULE_NAME -y
 
       - name: Run PHPCS
-        run: cd $HOME/drupal && vendor/bin/phpcs public/modules/custom/ --standard=Drupal
+        run: |
+          cd $DRUPAL_ROOT
+          vendor/bin/phpcs public/modules/custom/ --standard=Drupal
 
       - name: Run PHPUnit tests
         run: |
-          drush runserver $SIMPLETEST_BASE_URL --root $HOME/drupal > /dev/null 2>&1 &
+          cd $DRUPAL_ROOT
+          drush runserver $SIMPLETEST_BASE_URL > /dev/null 2>&1 &
           chromedriver --port=4444 > /dev/null 2>&1 &
-          cd $HOME/drupal && vendor/bin/phpunit -c $HOME/drupal/phpunit.xml.dist --verbose --group $MODULE_NAME
+          vendor/bin/phpunit -c $DRUPAL_ROOT/phpunit.xml.dist --verbose --group $MODULE_NAME

composer.json

@@ -12,6 +12,7 @@
     },
     "require-dev": {
         "dealerdirect/phpcodesniffer-composer-installer": "^0.7.0",
-        "drupal/coder": "^8.3"
+        "drupal/coder": "^8.3",
+        "phpspec/prophecy-phpunit": "^2"
     }
 }

config/install/language.content_settings.tpr_unit.tpr_unit.yml

@@ -0,0 +1,16 @@
+langcode: en
+status: true
+dependencies:
+  module:
+    - content_translation
+    - helfi_tpr
+third_party_settings:
+  content_translation:
+    enabled: true
+    bundle_settings:
+      untranslatable_fields_hide: '0'
+id: tpr_unit.tpr_unit
+target_entity_type_id: tpr_unit
+target_bundle: tpr_unit
+default_langcode: site_default
+language_alterable: true

config/install/migrate_plus.migration.tpr_unit.yml

@@ -1,3 +1,4 @@
+langcode: en
 status: true
 dependencies:
   enforced:
@@ -12,128 +13,44 @@ migration_tags:
 migration_group: tpr
 label: 'TPR unit'
 source:
-  plugin: url
-  urls: 'https://www.hel.fi/palvelukarttaws/rest/v4/unit/'
-  data_fetcher_plugin: http
-  data_parser_plugin: json
-  item_selector: 0
-  fields:
-    -
-      name: id
-      selector: /id
-    -
-      name: name
-      selector: /name_fi
-    -
-      name: call_charge_info
-      selector: /call_charge_info_fi
-    -
-      name: latitude
-      selector: /latitude
-    -
-      name: longitude
-      selector: /longitude
-    -
-      name: phone
-      selector: /phone
-    -
-      name: email
-      selector: /email
-    -
-      name: short_desc
-      selector: /short_desc_fi
-    -
-      name: desc
-      selector: /desc_fi
-    -
-      name: street_address
-      selector: /street_address_fi
-    -
-      name: address_city
-      selector: /address_city_fi
-    -
-      name: address_postal_code
-      selector: /address_zip
-    -
-      name: accessibility_email
-      selector: /accessibility_email
-    -
-      name: accessibility_phone
-      selector: /accessibility_phone
-    -
-      name: accessibility_www
-      selector: /accessibility_www
-    -
-      name: address_postal
-      selector: /address_postal_full_fi
-    -
-      name: www
-      selector: /www_fi
-    -
-      name: streetview_entrance_url
-      selector: /streetview_entrance_url
-    -
-      name: created
-      selector: /created_time
-    -
-      name: changed
-      selector: /modified_time
+  plugin: tpr
+  track_changes: true
+  url: 'https://www.hel.fi/palvelukarttaws/rest/v4/unit/'
   ids:
     id:
       type: string
 process:
   id: id
-  name:
-    plugin: substr
-    source: name
-    start: 0
-    length: 255
-  langcode:
+  latitude: latitude
+  longitude: longitude
+  streetview_entrance_url/uri: streetview_entrance_url
+  description/format:
     plugin: default_value
-    default_value: fi
-  address/postal_code: address_postal_code
-  address/locality: address_city
-  address/address_line1: street_address
+    default_value: filtered_html
+  address/postal_code: address_zip
   address/country_code:
     plugin: default_value
     default_value: FI
-  description/value: desc
-  description/summary: short_desc
-  description/format:
-    plugin: default_value
-    default_value: filtered_html
-  latitude: latitude
-  longitude: longitude
-  accessibility_email: accessibility_email
-  accessibility_phone: accessibility_phone
-  accessibility_www: accessibility_www
-  address_postal: address_postal
-  www/uri: www
-  streetview_entrance_url/uri: streetview_entrance_url
   phone:
     plugin: explode
     source: phone
     delimiter: ','
-  call_charge_info: call_charge_info
-  email: email
+    strict: false
   changed:
     -
       plugin: format_date
-      source: changed
+      source: modified_time
       from_format: 'Y-m-d\TH:i:s'
-      to_format: 'U'
-      from_timezone: 'Europe/Helsinki'
-      to_timezone: 'UTC'
-    -
-      plugin: entity_has_changed
-      entity_type: tpr_unit
+      to_format: U
+      from_timezone: Europe/Helsinki
+      to_timezone: UTC
   created:
     plugin: format_date
-    source: created
+    source: created_time
     from_format: 'Y-m-d\TH:i:s'
-    to_format: 'U'
-    from_timezone: 'Europe/Helsinki'
-    to_timezone: 'UTC'
+    to_format: U
+    from_timezone: Europe/Helsinki
+    to_timezone: UTC
 destination:
-  plugin: 'entity:tpr_unit'
-migration_dependencies: []
+  plugin: tpr_unit
+migration_dependencies: {  }