Skip to content

Commit

Permalink
Release 3.3.1d (#992)
Browse files Browse the repository at this point in the history 
Connector :
* Airtable : manage field id with read action
* Airtable : manage array fields
* SugarCRM : fix bug with deleted record
* Moodle : add groups and group_members modules

Bugfix : fix order data with the read limit is reached
  • Loading branch information
@Myddleware
Myddleware authored Apr 1, 2023
1 parent e17766c commit 771d278
Show file tree
Hide file tree
Showing 6 changed files with 68 additions and 39 deletions.
2 changes: 1 addition & 1 deletion .env
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
MYDDLEWARE_VERSION=3.3.1c
MYDDLEWARE_VERSION=3.3.1d

APP_SECRET=Thissecretisnotsosecretchangeit
APP_ENV=prod
Expand Down
4 changes: 2 additions & 2 deletions src/Manager/RuleManager.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -560,7 +560,7 @@ protected function validateReadDataSource()

// Order data in the date_modified order
$modified = array_column($dataSourceValues, 'date_modified');
array_multisort($modified, SORT_DESC, $dataSourceValues);
array_multisort($modified, SORT_ASC, $dataSourceValues);
foreach ($dataSourceValues as $value) {
// Check if the previous record has the same date_modified than the current record
// Check only if offset isn't managed into the source application connector
Expand Down Expand Up @@ -1413,7 +1413,7 @@ protected function massIdRerun(string $documentIds)
}
}
if (in_array($status, ['Predecessor_OK', 'Relate_KO'])) {
$response = $this->ckeckParentDocuments($arrayIdDocument);
$response = $this->checkParentDocument($arrayIdDocument);
if (true === $this->verifyMultiIdResponse($response)) {
// Update status if an action has been executed
$status = 'Relate_OK';
Expand Down
70 changes: 38 additions & 32 deletions src/Solutions/airtable.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ class airtablecore extends solution
protected string $airtableURL = 'https://api.airtable.com/v0/';
protected string $metadataApiEndpoint = 'https://api.airtable.com/v0/meta/bases/';

protected array $required_fields = ['default' => ['createdTime', 'Last Modified']];
/**
* Airtable base.
*/
Expand Down Expand Up @@ -188,20 +187,15 @@ public function readData($param): array
$param['fields'] = $this->cleanMyddlewareElementId($param['fields']);
// Add required fields
$param['fields'] = $this->addRequiredField($param['fields'], $param['module']);
// There is a bug on the parameter returnFieldsByFieldId soit can't be used
// In case we use fieldsId, we need to get the label to compare with Airtable result (only field label are returned)
include 'lib/airtable/metadata.php';
foreach ($param['fields'] as $field) {
if ('fld' == substr($field, 0, 3)) {
if (!empty($moduleFields[$baseID][$param['module']][$field]['label'])) {
$fields[$field] = $moduleFields[$baseID][$param['module']][$field]['label'];
continue;
}
}
$fields[$field] = $field;
}
// Get the reference date field name
$dateRefField = $this->getDateRefName($param['module'], $param['ruleParams']['mode']);

// Get the reference date field name only when we read using reference date
if (empty($param['query'])) {
$dateRefField = $this->getDateRefName($param['module'], $param['ruleParams']['mode']);
// Add the dateRefField in teh field list
if (array_search($dateRefField, $param['fields']) === false) {
$param['fields'][] = $dateRefField;
}
}
$stop = false;
$page = 1;
$offset = '';
Expand All @@ -213,7 +207,7 @@ public function readData($param): array
if (!empty($param['query'])) {
if (!empty($param['query']['id'])) {
$id = $param['query']['id'];
$response = $client->request('GET', $this->airtableURL.$baseID.'/'.$param['module'].'/'.$id, $options);
$response = $client->request('GET', $this->airtableURL.$baseID.'/'.$param['module'].'/'.$id.'?returnFieldsByFieldId=true', $options);
$statusCode = $response->getStatusCode();
$contentType = $response->getHeaders()['content-type'][0];
$content2 = $response->getContent();
Expand Down Expand Up @@ -253,7 +247,7 @@ public function readData($param): array
$filterByFormula .= ')';
}
// Get all records corresponding to the filters
$response = $client->request('GET', $this->airtableURL.$baseID.'/'.$param['module'].'?'.$filterByFormula, $options);
$response = $client->request('GET', $this->airtableURL.$baseID.'/'.$param['module'].'?returnFieldsByFieldId=true&'.$filterByFormula, $options);
$statusCode = $response->getStatusCode();
$contentType = $response->getHeaders()['content-type'][0];
$content = $response->getContent();
Expand All @@ -262,7 +256,7 @@ public function readData($param): array
} else {
// all records
$dateRef = $this->dateTimeFromMyddleware($param['date_ref']);
$response = $client->request('GET', $this->airtableURL.$baseID.'/'.$param['module']."?sort[0][field]=Last Modified&filterByFormula=IS_AFTER({Last Modified},'$dateRef')&pageSize=".$this->defaultLimit.'&maxRecords='.$param['limit'].$offset, $options);
$response = $client->request('GET', $this->airtableURL.$baseID.'/'.$param['module']."?sort[0][field]=Last Modified&filterByFormula=IS_AFTER({Last Modified},'$dateRef')&returnFieldsByFieldId=true&pageSize=".$this->defaultLimit.'&maxRecords='.$param['limit'].$offset, $options);
$statusCode = $response->getStatusCode();
$contentType = $response->getHeaders()['content-type'][0];
$content = $response->getContent();
Expand All @@ -277,19 +271,18 @@ public function readData($param): array
$content = $this->convertResponse($param, $content['records']);
foreach ($content as $record) {
++$currentCount;
foreach ($fields as $key => $field) {
$fieldWithSpace = str_replace('___', ' ', $field);
if (isset($record['fields'][$fieldWithSpace])) {
// Depending on the field type, the result can be an array, in this case we take the first result
if (is_array($record['fields'][$fieldWithSpace])) {
$result['values'][$record['id']][$key] = current($record['fields'][$fieldWithSpace]);
} else {
$result['values'][$record['id']][$key] = $record['fields'][$fieldWithSpace];
}
} else {
$result['values'][$record['id']][$key] = '';
foreach ($param['fields'] as $field) {
if (!empty($record['fields'][$field])) {
// If teh value is an array (relation), we take the first entry
if (is_array($record['fields'][$field])) {
$result['values'][$record['id']][$field] = $record['fields'][$field][0];
} else {
$result['values'][$record['id']][$field] = $record['fields'][$field];
}
} else {
$result['values'][$record['id']][$field] = '';
}
}
}

// Get the reference date
if (!empty($record['fields'][$dateRefField])) {
Expand Down Expand Up @@ -328,7 +321,6 @@ public function readData($param): array
$result['error'] = 'Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )';
$this->logger->error($e->getMessage().' '.$e->getFile().' '.$e->getLine());
}

return $result;
}

Expand Down Expand Up @@ -524,7 +516,21 @@ protected function convertResponse($param, $response)

public function getDateRefName($moduleSource, $ruleMode): string
{
return 'Last Modified';
// Search the field id
include 'lib/airtable/metadata.php';
$found_key = array_search('Last Modified', array_column($moduleFields[$this->paramConnexion['projectid']][$moduleSource], 'label'), true);
// Error if not found
if ($found_key === false) {
throw new Exception('Failed to found the date reference field name in the metadata');
}

// Get the field id
$fieldId = key(array_slice($moduleFields[$this->paramConnexion['projectid']][$moduleSource], $found_key, $found_key));
if (empty($fieldId)) {
throw new Exception('Failed to found the id corresponding to the date reference field.');
}

return $fieldId;
}

/**
Expand Down
4 changes: 3 additions & 1 deletion src/Solutions/lib/moodle/metadata.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,9 @@
],

'group_members' => [
'groupid' => ['label' => 'Group ID', 'type' => 'varchar(255)', 'type_bdd' => 'varchar(255)', 'required' => 0, 'required_relationship' => 1, 'relate' => true],
'id' => ['label' => 'ID', 'type' => 'varchar(255)', 'type_bdd' => 'varchar(255)', 'required' => 0],
'groupid' => ['label' => 'Group ID', 'type' => 'varchar(255)', 'type_bdd' => 'varchar(255)', 'required' => 0, 'required_relationship' => 1, 'relate' => true],
'userid' => ['label' => 'User ID', 'type' => 'varchar(255)', 'type_bdd' => 'varchar(255)', 'required' => 0, 'required_relationship' => 1, 'relate' => true],
'timeadded' => ['label' => 'Time added', 'type' => 'varchar(255)', 'type_bdd' => 'varchar(255)', 'required' => 0],
],
];
11 changes: 11 additions & 0 deletions src/Solutions/moodle.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ class moodlecore extends solution
'get_users_last_access' => ['id', 'lastaccess'],
'get_course_completion_by_date' => ['id', 'timecompleted'],
'get_user_grades' => ['id', 'timemodified'],
'groups' => ['id', 'timemodified'],
'group_members' => ['id', 'timeadded'],
];

protected array $FieldsDuplicate = [
Expand Down Expand Up @@ -116,6 +118,8 @@ public function get_modules($type = 'source'): array
'get_user_compentencies_by_date' => 'Get user compentency',
'get_competency_module_completion_by_date' => 'Get compentency module completion',
'get_user_grades' => 'Get user grades',
'groups' => 'Groups',
'group_members' => 'Group members',
];
}

Expand Down Expand Up @@ -584,6 +588,10 @@ protected function getFunctionName($param): string
return 'local_myddleware_get_users_by_date';
} elseif ('courses' == $param['module']) {
return 'local_myddleware_get_courses_by_date';
} elseif ('groups' == $param['module']) {
return 'local_myddleware_get_groups_by_date';
} elseif ('group_members' == $param['module']) {
return 'local_myddleware_get_group_members_by_date';
}
}
// In all other cases
Expand Down Expand Up @@ -638,6 +646,9 @@ public function getRefFieldName($param): string
return 'timemodified';
}
break;
case 'group_members':
return 'timeadded';
break;
default:
return 'timemodified';
break;
Expand Down
16 changes: 13 additions & 3 deletions src/Solutions/sugarcrm.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,8 @@ public function read($param)
$deleted = true;
$param['fields'][] = 'deleted';
}
// Flag to know if Myddleware has read only deleted records
$onlyDeletion = true;

// Init search parameters
$filterArgs = [
Expand Down Expand Up @@ -336,7 +338,11 @@ public function read($param)
and !empty($record->deleted)
) {
$result[$record->id]['myddleware_deletion'] = true;
}
} else {
// At least one non deleted record read
$onlyDeletion = false;
}

foreach ($param['fields'] as $field) {
// Sugar returns multilist value as array
if (
Expand All @@ -352,11 +358,15 @@ public function read($param)
}
$result[$record->id][$field] = (!empty($record->$field) ? $record->$field : '');
}
// No data returned if record deleted, we set a default date (date ref to have no impact on the final date ref)
// No date modified returned if record deleted, we set a default date (the last reference date read)
if (!empty($result[$record->id]['myddleware_deletion'])) {
$result[$record->id]['date_modified'] = $param['date_ref'];
$result[$record->id]['date_modified'] = end($records)->date_modified;
}
}
// Error if only deletion records read
if ($onlyDeletion) {
throw new \Exception('Only deletion records read. It is not possible to determine the reference date with only deletion. Please increase the rule limit to include non deletion records.');
}
}

return $result;
Expand Down

0 comments on commit 771d278

Please sign in to comment.