From 180a8014c45280c6bbf81f598ea791c775629448 Mon Sep 17 00:00:00 2001 From: colemanw Date: Fri, 6 Dec 2024 14:22:26 -0500 Subject: [PATCH 1/3] Add ImportTemplateField entity --- CRM/Core/DAO/ImportTemplateField.php | 12 +++ Civi/Api4/ImportTemplateField.php | 25 ++++++ .../Core/ImportTemplateField.entityType.php | 76 +++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 CRM/Core/DAO/ImportTemplateField.php create mode 100644 Civi/Api4/ImportTemplateField.php create mode 100644 schema/Core/ImportTemplateField.entityType.php diff --git a/CRM/Core/DAO/ImportTemplateField.php b/CRM/Core/DAO/ImportTemplateField.php new file mode 100644 index 000000000000..26e12c620884 --- /dev/null +++ b/CRM/Core/DAO/ImportTemplateField.php @@ -0,0 +1,12 @@ + 'ImportTemplateField', + 'table' => 'civicrm_import_template_field', + 'class' => 'CRM_Core_DAO_ImportTemplateField', + 'getInfo' => fn() => [ + 'title' => ts('Import Template Field'), + 'title_plural' => ts('Import Template Fields'), + 'description' => ts('Individual field for an import template'), + 'add' => '5.83', + ], + 'getFields' => fn() => [ + 'id' => [ + 'title' => ts('Field ID'), + 'sql_type' => 'int unsigned', + 'input_type' => 'Number', + 'required' => TRUE, + 'add' => '5.83', + 'primary_key' => TRUE, + 'auto_increment' => TRUE, + ], + 'user_job_id' => [ + 'title' => ts('Job ID'), + 'sql_type' => 'int unsigned', + 'input_type' => 'EntityRef', + 'required' => TRUE, + 'description' => ts('Template to which this field belongs'), + 'add' => '5.83', + 'input_attrs' => [ + 'label' => ts('Template'), + ], + 'entity_reference' => [ + 'entity' => 'UserJob', + 'key' => 'id', + 'on_delete' => 'CASCADE', + ], + ], + 'name' => [ + 'title' => ts('Field Name'), + 'sql_type' => 'varchar(1024)', + 'input_type' => 'Select', + 'description' => ts('Template field key'), + 'add' => '5.83', + ], + 'column_number' => [ + 'title' => ts('Column Number to map to'), + 'sql_type' => 'int unsigned', + 'input_type' => 'Number', + 'required' => TRUE, + 'description' => ts('Column number for the import dataset'), + 'add' => '5.83', + ], + 'entity' => [ + 'title' => ts('Entity'), + 'sql_type' => 'varchar(255)', + 'input_type' => 'Select', + 'description' => ts('Import entity'), + 'add' => '5.83', + ], + 'default_value' => [ + 'title' => ts('Default Value'), + 'sql_type' => 'varchar(1024)', + 'input_type' => 'Text', + 'add' => '5.83', + ], + 'data' => [ + 'title' => ts('Data'), + 'sql_type' => 'text', + 'description' => ts('Configuration data for the field'), + 'add' => '5.83', + 'default' => NULL, + 'serialize' => CRM_Core_DAO::SERIALIZE_JSON, + ], + ], +]; From abee4a6493d43c0ab23deca2b4027ec853fd58c1 Mon Sep 17 00:00:00 2001 From: colemanw Date: Tue, 31 Dec 2024 15:53:07 -0500 Subject: [PATCH 2/3] CiviImport - Switch import_mappings to ImportTemplateField table --- CRM/Contribute/Import/Parser/Contribution.php | 2 +- CRM/Import/Form/MapField.php | 4 +- CRM/Import/Forms.php | 67 +-------------- CRM/Import/Parser.php | 55 +----------- Civi/UserJob/UserJobTrait.php | 85 +++++++++++++++++++ ext/civiimport/ang/crmCiviimport.js | 22 +++-- 6 files changed, 106 insertions(+), 129 deletions(-) create mode 100644 Civi/UserJob/UserJobTrait.php diff --git a/CRM/Contribute/Import/Parser/Contribution.php b/CRM/Contribute/Import/Parser/Contribution.php index 5e006cf96444..a8081c0670ec 100644 --- a/CRM/Contribute/Import/Parser/Contribution.php +++ b/CRM/Contribute/Import/Parser/Contribution.php @@ -73,7 +73,7 @@ public static function getUserJobInfo(): array { * @throws \CRM_Core_Exception */ protected function getFieldMappings(): array { - $mappedFields = $this->getUserJob()['metadata']['import_mappings'] ?? []; + $mappedFields = $this->getUserJob()['template_fields'] ?? []; if (empty($mappedFields)) { foreach ($this->getSubmittedValue('mapper') as $i => $mapperRow) { $mappedField = $this->getMappingFieldFromMapperInput($mapperRow, 0, $i); diff --git a/CRM/Import/Form/MapField.php b/CRM/Import/Form/MapField.php index b5f372fd5076..8b7220296796 100644 --- a/CRM/Import/Form/MapField.php +++ b/CRM/Import/Form/MapField.php @@ -234,12 +234,12 @@ protected function getMappedField(array $fieldMapping, int $mappingID, int $colu * @throws \CRM_Core_Exception */ protected function saveMappingField(int $mappingID, int $columnNumber, bool $isUpdate = FALSE): void { - if (!empty($this->userJob['metadata']['import_mappings'])) { + if (!empty($this->userJob['template_fields'])) { // In this case Civi-Import has already saved the mapping to civicrm_user_job.metadata // and the code here is just keeping civicrm_mapping_field in sync. // Eventually we hope to phase out the use of the civicrm_mapping data & // just use UserJob and Import Templates (UserJob records with 'is_template' = 1 - $mappedFieldData = $this->userJob['metadata']['import_mappings'][$columnNumber]; + $mappedFieldData = $this->userJob['template_fields'][$columnNumber]; $mappedField = array_intersect_key($mappedFieldData, array_fill_keys(['name', 'column_number', 'entity_data'], TRUE)); $mappedField['mapping_id'] = $mappingID; } diff --git a/CRM/Import/Forms.php b/CRM/Import/Forms.php index 2107ae607a2c..f1522660086f 100644 --- a/CRM/Import/Forms.php +++ b/CRM/Import/Forms.php @@ -25,23 +25,13 @@ * This class helps the forms within the import flow access submitted & parsed values. */ class CRM_Import_Forms extends CRM_Core_Form { - + use \Civi\UserJob\UserJobTrait; /** * @var int */ protected $templateID; - /** - * User job id. - * - * This is the primary key of the civicrm_user_job table which is used to - * track the import. - * - * @var int - */ - protected $userJobID; - /** * Name of the import mapping (civicrm_mapping). * @@ -86,36 +76,6 @@ public function getEntity() { return $this->controller->getEntity(); } - /** - * @return int|null - */ - public function getUserJobID(): ?int { - if (!$this->userJobID && $this->get('user_job_id')) { - $this->userJobID = $this->get('user_job_id'); - } - return $this->userJobID; - } - - /** - * Set user job ID. - * - * @param int $userJobID - */ - public function setUserJobID(int $userJobID): void { - $this->userJobID = $userJobID; - // This set allows other forms in the flow ot use $this->get('user_job_id'). - $this->set('user_job_id', $userJobID); - } - - /** - * User job details. - * - * This is the relevant row from civicrm_user_job. - * - * @var array - */ - protected $userJob; - /** * @var \CRM_Import_Parser */ @@ -131,25 +91,6 @@ public function setUserJobID(int $userJobID): void { */ protected $isQuickFormMode = TRUE; - /** - * Get User Job. - * - * API call to retrieve the userJob row. - * - * @return array - * - * @throws \CRM_Core_Exception - */ - protected function getUserJob(): array { - if (!$this->userJob) { - $this->userJob = UserJob::get() - ->addWhere('id', '=', $this->getUserJobID()) - ->execute() - ->first(); - } - return $this->userJob; - } - /** * Get submitted values stored in the user job. * @@ -566,7 +507,7 @@ protected function isUpdateTemplateJob(): bool { */ protected function getColumnHeaders(): array { $headers = $this->getDataSourceObject()->getColumnHeaders(); - $mappedFields = $this->getUserJob()['metadata']['import_mappings'] ?? []; + $mappedFields = $this->getUserJob()['template_fields'] ?? []; if (!empty($mappedFields) && count($mappedFields) > count($headers)) { // The user has mapped one or more non-database fields, add those in. $userMappedFields = array_diff_key($mappedFields, $headers); @@ -608,7 +549,7 @@ protected function getDataRows($statuses = [], int $limit = 0): array { $statuses = (array) $statuses; $rows = $this->getDataSourceObject()->setLimit($limit)->setStatuses($statuses)->getRows(); $headers = $this->getColumnHeaders(); - $mappings = $this->getUserJob()['metadata']['import_mappings'] ?? []; + $mappings = $this->getUserJob()['template_fields'] ?? []; foreach ($rows as &$row) { foreach ($headers as $index => $header) { if (!$header) { @@ -843,7 +784,7 @@ protected function getParser() { protected function getMappedFieldLabels(): array { $mapper = []; $parser = $this->getParser(); - $importMappings = $this->getUserJob()['metadata']['import_mappings'] ?? []; + $importMappings = $this->getUserJob()['template_fields'] ?? []; if (empty($importMappings)) { foreach ($this->getSubmittedValue('mapper') as $columnNumber => $mapping) { $importMappings[] = $parser->getMappingFieldFromMapperInput((array) $mapping, 0, $columnNumber); diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php index a891382cae53..561e727766b9 100644 --- a/CRM/Import/Parser.php +++ b/CRM/Import/Parser.php @@ -30,6 +30,7 @@ * supported. */ abstract class CRM_Import_Parser implements UserJobInterface { + use \Civi\UserJob\UserJobTrait; /** * Return codes @@ -41,23 +42,6 @@ abstract class CRM_Import_Parser implements UserJobInterface { */ const DUPLICATE_SKIP = 1, DUPLICATE_UPDATE = 4, DUPLICATE_FILL = 8, DUPLICATE_NOCHECK = 16; - /** - * User job id. - * - * This is the primary key of the civicrm_user_job table which is used to - * track the import. - * - * @var int - */ - protected $userJobID; - - /** - * The user job in use. - * - * @var array - */ - protected $userJob; - /** * Potentially ambiguous options. * @@ -79,13 +63,6 @@ abstract class CRM_Import_Parser implements UserJobInterface { */ protected $siteDefaultCountry = NULL; - /** - * @return int|null - */ - public function getUserJobID(): ?int { - return $this->userJobID; - } - /** * Ids of contacts created this iteration. * @@ -93,17 +70,6 @@ public function getUserJobID(): ?int { */ protected $createdContacts = []; - /** - * Set user job ID. - * - * @param int $userJobID - * - * @return self - */ - public function setUserJobID(int $userJobID): self { - $this->userJobID = $userJobID; - return $this; - } /** * Countries that the site is restricted to @@ -120,25 +86,6 @@ public function getTrackingFields(): array { return []; } - /** - * Get User Job. - * - * API call to retrieve the userJob row. - * - * @return array - * - * @throws \CRM_Core_Exception - */ - protected function getUserJob(): array { - if (empty($this->userJob)) { - $this->userJob = UserJob::get() - ->addWhere('id', '=', $this->getUserJobID()) - ->execute() - ->first(); - } - return $this->userJob; - } - /** * Get the relevant datasource object. * diff --git a/Civi/UserJob/UserJobTrait.php b/Civi/UserJob/UserJobTrait.php new file mode 100644 index 000000000000..44e3af15d7eb --- /dev/null +++ b/Civi/UserJob/UserJobTrait.php @@ -0,0 +1,85 @@ +userJobID && is_a($this, 'CRM_Core_Form') && $this->get('user_job_id')) { + $this->userJobID = $this->get('user_job_id'); + } + return $this->userJobID; + } + + /** + * Set user job ID. + * + * @param int $userJobID + * + * @return self + */ + public function setUserJobID(int $userJobID): self { + $this->userJobID = $userJobID; + // This allows other forms in the flow ot use $this->get('user_job_id'). + if (is_a($this, 'CRM_Core_Form')) { + $this->set('user_job_id', $userJobID); + } + return $this; + } + + /** + * Get User Job. + * + * API call to retrieve the userJob row. + * + * @return array + * + * @throws \CRM_Core_Exception + */ + protected function getUserJob(): array { + if (empty($this->userJob)) { + $this->userJob = UserJob::get() + ->addWhere('id', '=', $this->getUserJobID()) + ->addChain('template_fields', ImportTemplateField::get() + ->addWhere('user_job_id', '=', $this->getUserJobID()) + ->addOrderBy('column_number') + ) + ->execute() + ->single(); + } + return $this->userJob; + } + +} diff --git a/ext/civiimport/ang/crmCiviimport.js b/ext/civiimport/ang/crmCiviimport.js index d862f93f2e8a..a4b2a3e16e11 100644 --- a/ext/civiimport/ang/crmCiviimport.js +++ b/ext/civiimport/ang/crmCiviimport.js @@ -2,10 +2,6 @@ // Declare a list of dependencies. angular.module('crmCiviimport', CRM.angRequires('crmCiviimport')); - // The controller uses *injection*. This default injects a few things: - // $scope -- This is the set of variables shared between JS and HTML. - // crmApi, crmStatus, crmUiHelp -- These are services provided by civicrm-core. - // myContact -- The current contact, defined above in config(). angular.module('crmCiviimport').component('crmImportUi', { templateUrl: '~/crmCiviimport/Import.html', controller: function($scope, crmApi4, crmStatus, crmUiHelp) { @@ -72,7 +68,7 @@ function buildImportMappings() { $scope.data.importMappings = []; - var importMappings = $scope.userJob.metadata.import_mappings; + var importMappings = $scope.userJob.template_fields; _.each($scope.data.columnHeaders, function (header, index) { var fieldName = $scope.data.defaults['mapper[' + index + ']'][0]; if (Boolean(fieldName)) { @@ -280,7 +276,7 @@ $scope.save = (function ($event) { $event.preventDefault(); $scope.userJob.metadata.entity_configuration = {}; - $scope.userJob.metadata.import_mappings = []; + $scope.userJob.template_fields = []; _.each($scope.entitySelection, function (entity) { $scope.userJob.metadata.entity_configuration[entity.id] = entity.selected; }); @@ -294,15 +290,23 @@ entityConfig = {'soft_credit': $scope.userJob.metadata.entity_configuration[selectedEntity]}; } - $scope.userJob.metadata.import_mappings.push({ + $scope.userJob.template_fields.push({ name: importRow.selectedField, default_value: importRow.defaultValue, // At this stage column_number is thrown away but we store it here to have it for when we change that. - column_number: index, + column_number: index + 1, entity_data: entityConfig }); }); - crmApi4('UserJob', 'save', {records: [$scope.userJob]}) + crmApi4('UserJob', 'save', { + records: [$scope.userJob], + chain: { + template_fields: ['ImportTemplateField', 'replace', { + where: [['user_job_id', '=', '$id']], + records: $scope.userJob.template_fields, + }], + }, + }) .then(function(result) { // Only post the form if the save succeeds. document.getElementById("MapField").submit(); From 1dbd8e4b566c74ad6787428595637b0d535c35ca Mon Sep 17 00:00:00 2001 From: colemanw Date: Sun, 5 Jan 2025 18:14:18 -0500 Subject: [PATCH 3/3] ImportTemplateField - Add pseudoconstants --- CRM/Core/BAO/ImportTemplateField.php | 66 +++++++++++++++++++ .../Core/ImportTemplateField.entityType.php | 4 ++ 2 files changed, 70 insertions(+) create mode 100644 CRM/Core/BAO/ImportTemplateField.php diff --git a/CRM/Core/BAO/ImportTemplateField.php b/CRM/Core/BAO/ImportTemplateField.php new file mode 100644 index 000000000000..4ccabdfc3417 --- /dev/null +++ b/CRM/Core/BAO/ImportTemplateField.php @@ -0,0 +1,66 @@ + 'save', + 'select' => ['name', 'label', 'description'], + 'where' => [ + ['usage', 'CONTAINS', 'import'], + ], + ]); + } + + /** + * Pseudoconstant callback for the 'entity' field + */ + public static function getImportableEntityOptions(string $fieldName, array $params):? array { + $values = $params['values']; + $userJobId = $values['user_job_id'] ?? NULL; + if (!$userJobId && !empty($values['id'])) { + $userJobId = CRM_Core_BAO_UserJob::getDbVal('user_job_id', $values['id']); + } + + if (!$userJobId) { + return NULL; + } + $entities = []; + $jobTypes = array_column(CRM_Core_BAO_UserJob::getTypes(), NULL, 'id'); + $jobType = CRM_Core_BAO_UserJob::getDbVal('job_type', $values['user_job_id']); + + $mainEntityName = $jobTypes[$jobType]['entity'] ?? NULL; + // TODO: For now each job type only supports one entity, + // so this select list doesn't (yet) have more than one option. + $entities[] = [ + 'id' => $mainEntityName, + 'name' => $mainEntityName, + 'label' => CoreUtil::getInfoItem($mainEntityName, 'title'), + 'icon' => CoreUtil::getInfoItem($mainEntityName, 'icon'), + ]; + + return $entities; + } + +} diff --git a/schema/Core/ImportTemplateField.entityType.php b/schema/Core/ImportTemplateField.entityType.php index 683e8b449aad..052dae578eb2 100644 --- a/schema/Core/ImportTemplateField.entityType.php +++ b/schema/Core/ImportTemplateField.entityType.php @@ -41,6 +41,10 @@ 'sql_type' => 'varchar(1024)', 'input_type' => 'Select', 'description' => ts('Template field key'), + 'pseudoconstant' => [ + 'callback' => ['CRM_Core_BAO_ImportTemplateField', 'getImportableFieldOptions'], + 'suffixes' => ['name', 'label', 'description'], + ], 'add' => '5.83', ], 'column_number' => [