From dd0d2a4319ffd458f0ae1c09f00b39c4a4434050 Mon Sep 17 00:00:00 2001 From: sam marshall Date: Wed, 23 Aug 2023 15:31:19 +0100 Subject: [PATCH 01/28] Removed legacy cron function This change removes the old cron function in lib.php which is no longer needed in Moodle 4.2. --- lib.php | 16 ---------------- tests/local/tasks_test.php | 7 ------- 2 files changed, 23 deletions(-) diff --git a/lib.php b/lib.php index 59159ebe..5106bcc3 100644 --- a/lib.php +++ b/lib.php @@ -69,22 +69,6 @@ define('TOOL_OBJECTFS_DELETE_EXTERNAL_TRASH', 1); define('TOOL_OBJECTFS_DELETE_EXTERNAL_FULL', 2); -// Legacy cron function. -function tool_objectfs_cron() { - mtrace('RUNNING legacy cron objectfs'); - global $CFG; - if ($CFG->branch <= 26) { - // Unlike the task system, we do not get fine grained control over - // when tasks/manipulators run. Every cron we just run all the manipulators. - (new manipulator_builder())->execute_all(); - - \tool_objectfs\local\report\objectfs_report::cleanup_reports(); - \tool_objectfs\local\report\objectfs_report::generate_status_report(); - } - - return true; -} - /** * Sends a plugin file to the browser. * @param $course diff --git a/tests/local/tasks_test.php b/tests/local/tasks_test.php index 6cae7841..8a71d024 100644 --- a/tests/local/tasks_test.php +++ b/tests/local/tasks_test.php @@ -32,13 +32,6 @@ protected function tearDown(): void { ob_end_clean(); } - public function test_run_legacy_cron() { - $config = manager::get_objectfs_config(); - $config->enabletasks = true; - manager::set_objectfs_config($config); - $this->assertTrue(tool_objectfs_cron()); - } - public function test_run_scheduled_tasks() { global $CFG; // If tasks not implemented. From 0b283e9409fd2a365f397f9324d1104243bb5bc4 Mon Sep 17 00:00:00 2001 From: sam marshall Date: Wed, 22 Nov 2023 13:52:39 +0000 Subject: [PATCH 02/28] Fix PHP 8.2 deprecation warnings in unit tests There were a number of 'Creation of dynamic property...' deprecation warnings when running the unit test. These happen if you set '$this->field = value' without declaring e.g. 'protected $field' in the class. I've fixed these and the unit tests now run for me. The fix should be safe for all recent PHP versions, i.e. I didn't add types to the declarations. --- classes/local/store/s3/client.php | 3 +++ classes/tests/testcase.php | 6 ++++++ tests/local/object_manipulator/checker_test.php | 3 +++ tests/local/object_manipulator/deleter_test.php | 3 +++ tests/local/object_manipulator/orphaner_test.php | 3 +++ tests/local/object_manipulator/puller_test.php | 3 +++ tests/local/object_manipulator/pusher_test.php | 3 +++ tests/local/object_manipulator/recoverer_test.php | 6 ++++++ 8 files changed, 30 insertions(+) diff --git a/classes/local/store/s3/client.php b/classes/local/store/s3/client.php index 31288859..12ae1de0 100644 --- a/classes/local/store/s3/client.php +++ b/classes/local/store/s3/client.php @@ -48,6 +48,9 @@ class client extends object_client_base { protected $bucket; private $signingmethod; + /** @var string Prefix for bucket keys */ + protected $bucketkeyprefix; + public function __construct($config) { global $CFG; $this->autoloader = $CFG->dirroot . '/local/aws/sdk/aws-autoloader.php'; diff --git a/classes/tests/testcase.php b/classes/tests/testcase.php index 4b39f5a6..0d553dbb 100644 --- a/classes/tests/testcase.php +++ b/classes/tests/testcase.php @@ -27,6 +27,12 @@ abstract class testcase extends \advanced_testcase { + /** @var test_file_system Filesystem */ + public $filesystem; + + /** @var \tool_objectfs\log\objectfs_logger Logger */ + public $logger; + protected function setUp(): void { global $CFG; $CFG->alternative_file_system_class = '\\tool_objectfs\\tests\\test_file_system'; diff --git a/tests/local/object_manipulator/checker_test.php b/tests/local/object_manipulator/checker_test.php index 64f7866a..32385996 100644 --- a/tests/local/object_manipulator/checker_test.php +++ b/tests/local/object_manipulator/checker_test.php @@ -28,6 +28,9 @@ class checker_test extends \tool_objectfs\tests\testcase { /** @var string $manipulator */ protected $manipulator = checker::class; + /** @var checker Checker */ + protected $checker; + protected function setUp(): void { parent::setUp(); $config = manager::get_objectfs_config(); diff --git a/tests/local/object_manipulator/deleter_test.php b/tests/local/object_manipulator/deleter_test.php index 2e1726f9..2588d999 100644 --- a/tests/local/object_manipulator/deleter_test.php +++ b/tests/local/object_manipulator/deleter_test.php @@ -28,6 +28,9 @@ class deleter_test extends \tool_objectfs\tests\testcase { /** @var string $manipulator */ protected $manipulator = deleter::class; + /** @var deleter Deleter object */ + protected $deleter; + protected function setUp(): void { parent::setUp(); $config = manager::get_objectfs_config(); diff --git a/tests/local/object_manipulator/orphaner_test.php b/tests/local/object_manipulator/orphaner_test.php index a692eac9..e5f7a04b 100644 --- a/tests/local/object_manipulator/orphaner_test.php +++ b/tests/local/object_manipulator/orphaner_test.php @@ -29,6 +29,9 @@ class orphaner_test extends \tool_objectfs\tests\testcase { /** @var string $manipulator */ protected $manipulator = orphaner::class; + /** @var orphaner Orphaner object */ + protected $orphaner; + protected function setUp(): void { parent::setUp(); $config = manager::get_objectfs_config(); diff --git a/tests/local/object_manipulator/puller_test.php b/tests/local/object_manipulator/puller_test.php index bf5986c4..04b6e5da 100644 --- a/tests/local/object_manipulator/puller_test.php +++ b/tests/local/object_manipulator/puller_test.php @@ -28,6 +28,9 @@ class puller_test extends \tool_objectfs\tests\testcase { /** @var string $manipulator */ protected $manipulator = puller::class; + /** @var puller Puller object */ + protected $puller; + protected function setUp(): void { parent::setUp(); $config = manager::get_objectfs_config(); diff --git a/tests/local/object_manipulator/pusher_test.php b/tests/local/object_manipulator/pusher_test.php index d306f79a..e7c1f3db 100644 --- a/tests/local/object_manipulator/pusher_test.php +++ b/tests/local/object_manipulator/pusher_test.php @@ -29,6 +29,9 @@ class pusher_test extends \tool_objectfs\tests\testcase { /** @var string $manipulator */ protected $manipulator = pusher::class; + /** @var pusher Pusher object */ + protected $pusher; + protected function setUp(): void { parent::setUp(); $config = manager::get_objectfs_config(); diff --git a/tests/local/object_manipulator/recoverer_test.php b/tests/local/object_manipulator/recoverer_test.php index 43f68526..8a4ec5ba 100644 --- a/tests/local/object_manipulator/recoverer_test.php +++ b/tests/local/object_manipulator/recoverer_test.php @@ -26,6 +26,12 @@ */ class recoverer_test extends \tool_objectfs\tests\testcase { + /** @var candidates_finder Candidates finder object */ + protected $candidatesfinder; + + /** @var recoverer Recoverer object */ + protected $recoverer; + protected function setUp(): void { parent::setUp(); $config = manager::get_objectfs_config(); From 6cbf0ac0100afb96e75e3d520b4e7e98fd37597d Mon Sep 17 00:00:00 2001 From: Eric Phetteplace Date: Fri, 1 Mar 2024 10:38:41 -0800 Subject: [PATCH 03/28] Update README.md Using Google Cloud storage also requires the AWS SDK --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 88c0c6b2..3f52804f 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ This plugin is GDPR complient if you enable the deletion of remote objects. 2. Setup your remote object storage. See [Remote object storage setup](#amazon-s3) 3. Clone this repository into admin/tool/objectfs 4. Install one of the required SDK libraries for the storage file system that you will be using - 1. Clone [moodle-local_aws](https://github.com/catalyst/moodle-local_aws) into local/aws for S3 or DigitalOcean Spaces, or + 1. Clone [moodle-local_aws](https://github.com/catalyst/moodle-local_aws) into local/aws for S3 or DigitalOcean Spaces or Google Cloud, or 2. Clone [moodle-local_azure_storage](https://github.com/catalyst/moodle-local_azure_storage) into local/azure_storage for Azure Blob Storage, or 3. Clone [moodle-local_openstack](https://github.com/matt-catalyst/moodle-local_openstack.git) into local/openstack for openstack(swift) storage 5. Install the plugins through the moodle GUI. From 435d8dbc6edbf4217ab17b2483aa88ac5a9fc33c Mon Sep 17 00:00:00 2001 From: Dan Marsden Date: Tue, 30 Jul 2024 09:20:04 +1200 Subject: [PATCH 04/28] correct details in version.php --- version.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version.php b/version.php index bf606bc1..f53c4dc5 100644 --- a/version.php +++ b/version.php @@ -27,7 +27,7 @@ $plugin->version = 2023051701; // The current plugin version (Date: YYYYMMDDXX). $plugin->release = 2023051701; // Same as version. -$plugin->requires = 2020110900; // Requires Filesystem API. +$plugin->requires = 2023042400; // Requires 4.2 $plugin->component = "tool_objectfs"; $plugin->maturity = MATURITY_STABLE; -$plugin->supported = [310, 402]; +$plugin->supported = [402, 405]; From 37887673d5a9da6b2cd9c63d2107d7bfaa9760c5 Mon Sep 17 00:00:00 2001 From: Jay Oswald Date: Fri, 5 Jul 2024 16:11:56 +1000 Subject: [PATCH 05/28] Add phpdocs and resume checks (#593) * Re-instate workflow and add initial phpdoc * Add placeholder phpdocs * linting * fix privacy * empty commit to re-run tests * add provider back --- .github/workflows/ci.yml | 2 - classes/azure_file_system.php | 3 + classes/digitalocean_file_system.php | 4 + classes/local/manager.php | 24 ++- .../candidates/candidates_factory.php | 7 +- .../candidates/candidates_finder.php | 5 + .../candidates/checker_candidates.php | 12 +- .../candidates/deleter_candidates.php | 12 +- .../candidates/manipulator_candidates.php | 1 + .../manipulator_candidates_base.php | 6 +- .../candidates/orphaner_candidates.php | 12 +- .../candidates/puller_candidates.php | 12 +- .../candidates/pusher_candidates.php | 14 +- .../candidates/recoverer_candidates.php | 12 +- classes/local/object_manipulator/checker.php | 4 + classes/local/object_manipulator/deleter.php | 5 + classes/local/object_manipulator/logger.php | 53 +++++- .../local/object_manipulator/manipulator.php | 4 + .../manipulator_builder.php | 6 + .../object_manipulator/object_manipulator.php | 2 + classes/local/object_manipulator/orphaner.php | 5 + classes/local/object_manipulator/puller.php | 4 + classes/local/object_manipulator/pusher.php | 4 + .../local/object_manipulator/recoverer.php | 4 + .../local/report/location_report_builder.php | 5 + .../local/report/log_size_report_builder.php | 15 ++ .../local/report/mime_type_report_builder.php | 9 + .../report/object_status_history_table.php | 3 + classes/local/report/objectfs_report.php | 30 +++- .../local/report/objectfs_report_builder.php | 9 + classes/local/store/azure/client.php | 48 +++++- classes/local/store/azure/file_system.php | 9 + classes/local/store/azure/stream_wrapper.php | 157 ++++++++++++++---- classes/local/store/digitalocean/client.php | 18 +- .../local/store/digitalocean/file_system.php | 10 ++ classes/local/store/object_client.php | 96 +++++++++++ classes/local/store/object_client_base.php | 26 +++ classes/local/store/object_file_system.php | 122 +++++++++++++- classes/local/store/s3/client.php | 75 ++++++++- classes/local/store/s3/file_system.php | 22 ++- classes/local/store/signed_url.php | 3 + classes/local/store/swift/client.php | 83 ++++++++- classes/local/store/swift/file_system.php | 9 + classes/local/store/swift/stream_wrapper.php | 23 +-- classes/local/table/files_table.php | 75 +++++++++ classes/log/aggregate_logger.php | 72 +++++++- classes/log/null_logger.php | 29 ++++ classes/log/objectfs_logger.php | 64 +++++++ classes/log/objectfs_statistic.php | 43 +++++ classes/log/real_time_logger.php | 72 ++++++++ classes/privacy/provider.php | 4 +- classes/s3_file_system.php | 3 + classes/swift_file_system.php | 3 + classes/task/check_objects_location.php | 3 + .../task/delete_local_empty_directories.php | 5 +- classes/task/delete_local_objects.php | 3 + .../task/delete_orphaned_object_metadata.php | 7 +- classes/task/generate_status_report.php | 4 + classes/task/orphan_objects.php | 6 +- classes/task/pull_objects_from_storage.php | 3 + classes/task/push_objects_to_storage.php | 3 + classes/task/recover_error_objects.php | 3 + classes/task/task.php | 4 + .../tests/test_azure_integration_client.php | 18 ++ classes/tests/test_client.php | 70 +++++++- .../test_digitalocean_integration_client.php | 18 ++ classes/tests/test_file_system.php | 13 ++ classes/tests/test_s3_integration_client.php | 17 ++ .../tests/test_swift_integration_client.php | 17 ++ classes/tests/testcase.php | 146 +++++++++++++++- db/upgrade.php | 6 + lib.php | 4 +- presignedurl_tests.php | 2 +- renderer.php | 16 ++ .../local/object_manipulator/checker_test.php | 1 + .../local/object_manipulator/deleter_test.php | 8 + .../object_manipulator/orphaner_test.php | 8 + .../local/object_manipulator/puller_test.php | 8 + .../local/object_manipulator/pusher_test.php | 8 + .../object_manipulator/recoverer_test.php | 1 + tests/local/tasks_test.php | 1 + tests/object_file_system_test.php | 33 +++- 82 files changed, 1666 insertions(+), 124 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9742ffb3..ecd6546a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,5 +6,3 @@ on: [push, pull_request] jobs: ci: uses: catalyst/catalyst-moodle-workflows/.github/workflows/ci.yml@main - with: - disable_phpdoc: true diff --git a/classes/azure_file_system.php b/classes/azure_file_system.php index 81374ab6..f5a693d2 100644 --- a/classes/azure_file_system.php +++ b/classes/azure_file_system.php @@ -27,6 +27,9 @@ use tool_objectfs\local\store\azure\file_system; +/** + * Unknown? + */ class azure_file_system extends file_system { } diff --git a/classes/digitalocean_file_system.php b/classes/digitalocean_file_system.php index c1859bfc..fab7527e 100644 --- a/classes/digitalocean_file_system.php +++ b/classes/digitalocean_file_system.php @@ -19,6 +19,7 @@ * * @package tool_objectfs * @author Brian Yanosik + * @copyright Catalyst IT * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ @@ -26,6 +27,9 @@ use tool_objectfs\local\store\digitalocean\file_system; +/** + * Unknown? + */ class digitalocean_file_system extends file_system { } diff --git a/classes/local/manager.php b/classes/local/manager.php index c746b9c2..5d791f97 100644 --- a/classes/local/manager.php +++ b/classes/local/manager.php @@ -28,12 +28,15 @@ use stdClass; use tool_objectfs\local\store\object_file_system; -defined('MOODLE_INTERNAL') || die(); - +/** + * [Description manager] + */ class manager { /** - * @param $config + * set_objectfs_config + * @param stdClass $config + * @return void */ public static function set_objectfs_config($config) { foreach ($config as $key => $value) { @@ -42,6 +45,7 @@ public static function set_objectfs_config($config) { } /** + * get_objectfs_config * @return stdClass * @throws \dml_exception */ @@ -114,8 +118,9 @@ public static function get_objectfs_config() { } /** - * @param $config - * @return bool + * get_client + * @param stdClass $config + * @return mixed|bool */ public static function get_client($config) { $clientclass = self::get_client_classname_from_fs($config->filesystem); @@ -128,8 +133,9 @@ public static function get_client($config) { } /** - * @param $contenthash - * @param $newlocation + * update_object_by_hash + * @param string $contenthash + * @param string $newlocation * @param int|null $filesize Size of the file in bytes. Falls back to stored value if not provided. * @return mixed|stdClass * @throws \dml_exception @@ -172,8 +178,9 @@ public static function update_object_by_hash($contenthash, $newlocation, $filesi } /** + * update_object * @param stdClass $object - * @param $newlocation + * @param string $newlocation * @return stdClass * @throws \dml_exception */ @@ -192,6 +199,7 @@ public static function update_object(stdClass $object, $newlocation) { } /** + * cloudfront_pem_exists * @return string * @throws \coding_exception * @throws \dml_exception diff --git a/classes/local/object_manipulator/candidates/candidates_factory.php b/classes/local/object_manipulator/candidates/candidates_factory.php index d177ea09..8a3d1892 100644 --- a/classes/local/object_manipulator/candidates/candidates_factory.php +++ b/classes/local/object_manipulator/candidates/candidates_factory.php @@ -33,6 +33,9 @@ use tool_objectfs\local\object_manipulator\recoverer; use tool_objectfs\local\object_manipulator\orphaner; +/** + * Candidates Factory + */ class candidates_factory { /** @var array $manipulatormap */ @@ -46,8 +49,10 @@ class candidates_factory { ]; /** - * @param $manipulator + * Finder + * @param mixed $manipulator * @param stdClass $config + * * @return mixed * @throws moodle_exception */ diff --git a/classes/local/object_manipulator/candidates/candidates_finder.php b/classes/local/object_manipulator/candidates/candidates_finder.php index 41d955d6..79cfea33 100644 --- a/classes/local/object_manipulator/candidates/candidates_finder.php +++ b/classes/local/object_manipulator/candidates/candidates_finder.php @@ -27,6 +27,9 @@ use moodle_exception; use stdClass; +/** + * Candidates Finder + */ class candidates_finder { /** @var string $finder */ @@ -43,6 +46,7 @@ public function __construct($manipulator, stdClass $config) { } /** + * get * @return array */ public function get() { @@ -50,6 +54,7 @@ public function get() { } /** + * get_query_name * @return string */ public function get_query_name() { diff --git a/classes/local/object_manipulator/candidates/checker_candidates.php b/classes/local/object_manipulator/candidates/checker_candidates.php index 5f2d9e82..ef2298b7 100644 --- a/classes/local/object_manipulator/candidates/checker_candidates.php +++ b/classes/local/object_manipulator/candidates/checker_candidates.php @@ -24,13 +24,19 @@ namespace tool_objectfs\local\object_manipulator\candidates; +/** + * chcker_candiates + */ class checker_candidates extends manipulator_candidates_base { - /** @var string $queryname */ + /** + * queryname + * @var string + */ protected $queryname = 'get_check_candidates'; /** - * @inheritDoc + * get_candiates_sql * @return string */ public function get_candidates_sql() { @@ -43,7 +49,7 @@ public function get_candidates_sql() { } /** - * @inheritDoc + * get_candidates_sql_params * @return array */ public function get_candidates_sql_params() { diff --git a/classes/local/object_manipulator/candidates/deleter_candidates.php b/classes/local/object_manipulator/candidates/deleter_candidates.php index e17a179e..30f7d2df 100644 --- a/classes/local/object_manipulator/candidates/deleter_candidates.php +++ b/classes/local/object_manipulator/candidates/deleter_candidates.php @@ -24,13 +24,19 @@ namespace tool_objectfs\local\object_manipulator\candidates; +/** + * deleter_candidates + */ class deleter_candidates extends manipulator_candidates_base { - /** @var string $queryname */ + /** + * queryname + * @var string + */ protected $queryname = 'get_delete_candidates'; /** - * @inheritDoc + * get_candiates_sql * @return string */ public function get_candidates_sql() { @@ -43,7 +49,7 @@ public function get_candidates_sql() { } /** - * @inheritDoc + * get_candiates_sql_params * @return array */ public function get_candidates_sql_params() { diff --git a/classes/local/object_manipulator/candidates/manipulator_candidates.php b/classes/local/object_manipulator/candidates/manipulator_candidates.php index 8782316c..eac7ca90 100644 --- a/classes/local/object_manipulator/candidates/manipulator_candidates.php +++ b/classes/local/object_manipulator/candidates/manipulator_candidates.php @@ -50,6 +50,7 @@ public function get_candidates_sql(); public function get_candidates_sql_params(); /** + * get * @return array * @throws dml_exception */ diff --git a/classes/local/object_manipulator/candidates/manipulator_candidates_base.php b/classes/local/object_manipulator/candidates/manipulator_candidates_base.php index b4bded73..4810b29d 100644 --- a/classes/local/object_manipulator/candidates/manipulator_candidates_base.php +++ b/classes/local/object_manipulator/candidates/manipulator_candidates_base.php @@ -27,6 +27,9 @@ use dml_exception; use stdClass; +/** + * manipulator_candidates_base + */ abstract class manipulator_candidates_base implements manipulator_candidates { /** @var stdClass $config */ @@ -41,7 +44,7 @@ public function __construct(stdClass $config) { } /** - * @inheritDoc + * get_query_name * @return string */ public function get_query_name() { @@ -49,6 +52,7 @@ public function get_query_name() { } /** + * get * @return array * @throws dml_exception */ diff --git a/classes/local/object_manipulator/candidates/orphaner_candidates.php b/classes/local/object_manipulator/candidates/orphaner_candidates.php index 8f5d3416..e50f4806 100644 --- a/classes/local/object_manipulator/candidates/orphaner_candidates.php +++ b/classes/local/object_manipulator/candidates/orphaner_candidates.php @@ -24,13 +24,19 @@ namespace tool_objectfs\local\object_manipulator\candidates; +/** + * orphaner_candidates + */ class orphaner_candidates extends manipulator_candidates_base { - /** @var string $queryname */ + /** + * queryname + * @var string + */ protected $queryname = 'get_orphan_candidates'; /** - * @inheritDoc + * get_candidates_sql * @return string */ public function get_candidates_sql() { @@ -42,7 +48,7 @@ public function get_candidates_sql() { } /** - * @inheritDoc + * get_candidates_sql_params * @return array */ public function get_candidates_sql_params() { diff --git a/classes/local/object_manipulator/candidates/puller_candidates.php b/classes/local/object_manipulator/candidates/puller_candidates.php index b31e0f93..86ddab65 100644 --- a/classes/local/object_manipulator/candidates/puller_candidates.php +++ b/classes/local/object_manipulator/candidates/puller_candidates.php @@ -24,13 +24,19 @@ namespace tool_objectfs\local\object_manipulator\candidates; +/** + * puller_candidates + */ class puller_candidates extends manipulator_candidates_base { - /** @var string $queryname */ + /** + * queryname + * @var string + */ protected $queryname = 'get_pull_candidates'; /** - * @inheritDoc + * get_candidates_sql * @return string */ public function get_candidates_sql() { @@ -42,7 +48,7 @@ public function get_candidates_sql() { } /** - * @inheritDoc + * get_candidates_sql_params * @return array */ public function get_candidates_sql_params() { diff --git a/classes/local/object_manipulator/candidates/pusher_candidates.php b/classes/local/object_manipulator/candidates/pusher_candidates.php index a685a975..4c967710 100644 --- a/classes/local/object_manipulator/candidates/pusher_candidates.php +++ b/classes/local/object_manipulator/candidates/pusher_candidates.php @@ -24,15 +24,19 @@ namespace tool_objectfs\local\object_manipulator\candidates; -defined('MOODLE_INTERNAL') || die(); - +/** + * pusher_candidates + */ class pusher_candidates extends manipulator_candidates_base { - /** @var string $queryname */ + /** + * queryname + * @var string + */ protected $queryname = 'get_push_candidates'; /** - * @inheritDoc + * get_candidates_sql * @return string */ public function get_candidates_sql() { @@ -46,7 +50,7 @@ public function get_candidates_sql() { } /** - * @inheritDoc + * get_candidates_sql_params * @return array */ public function get_candidates_sql_params() { diff --git a/classes/local/object_manipulator/candidates/recoverer_candidates.php b/classes/local/object_manipulator/candidates/recoverer_candidates.php index d4dfd81f..4d1945ec 100644 --- a/classes/local/object_manipulator/candidates/recoverer_candidates.php +++ b/classes/local/object_manipulator/candidates/recoverer_candidates.php @@ -24,13 +24,19 @@ namespace tool_objectfs\local\object_manipulator\candidates; +/** + * recoverer_candidates + */ class recoverer_candidates extends manipulator_candidates_base { - /** @var string $queryname */ + /** + * queryname + * @var string + */ protected $queryname = 'get_recover_candidates'; /** - * @inheritDoc + * get_candidates_sql * @return string */ public function get_candidates_sql() { @@ -41,7 +47,7 @@ public function get_candidates_sql() { } /** - * @inheritDoc + * get_candidates_sql_params * @return array */ public function get_candidates_sql_params() { diff --git a/classes/local/object_manipulator/checker.php b/classes/local/object_manipulator/checker.php index b727eabe..7daf330c 100644 --- a/classes/local/object_manipulator/checker.php +++ b/classes/local/object_manipulator/checker.php @@ -29,6 +29,9 @@ use tool_objectfs\local\store\object_file_system; use tool_objectfs\log\aggregate_logger; +/** + * checker + */ class checker extends manipulator { /** @@ -45,6 +48,7 @@ public function __construct(object_file_system $filesystem, stdClass $config, ag } /** + * manipulate_object * @param stdClass $objectrecord * @return int */ diff --git a/classes/local/object_manipulator/deleter.php b/classes/local/object_manipulator/deleter.php index bbaba341..283b9d5c 100644 --- a/classes/local/object_manipulator/deleter.php +++ b/classes/local/object_manipulator/deleter.php @@ -29,6 +29,9 @@ use tool_objectfs\local\store\object_file_system; use tool_objectfs\log\aggregate_logger; +/** + * deleter + */ class deleter extends manipulator { /** @@ -61,6 +64,7 @@ public function __construct(object_file_system $filesystem, stdClass $config, ag } /** + * manipulate_object * @param stdClass $objectrecord * @return int */ @@ -70,6 +74,7 @@ public function manipulate_object(stdClass $objectrecord) { } /** + * manipulator_can_execute * @return bool */ protected function manipulator_can_execute() { diff --git a/classes/local/object_manipulator/logger.php b/classes/local/object_manipulator/logger.php index 4c09f54b..9bb8e61a 100644 --- a/classes/local/object_manipulator/logger.php +++ b/classes/local/object_manipulator/logger.php @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Moodle. If not, see . -/* Logs manipulator actions +/** Logs manipulator actions * * @package tool_objectfs * @author Kenneth Hendricks @@ -28,36 +28,81 @@ require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); +/** + * logger + */ class logger { + /** + * @var [type] + */ private $action; // Which action to log. + /** + * @var int + */ private $timestart; + /** + * @var int + */ private $timeend; + /** + * @var int + */ private $totalfilesize; + /** + * @var int + */ private $totalfilecount; + /** + * construct + */ public function __construct() { $this->totalfilecount = 0; $this->totalfilesize = 0; } + /** + * start_timing + * @return void + */ public function start_timing() { $this->timestart = time(); } + /** + * end_timing + * @return void + */ public function end_timing() { $this->timeend = time(); } + /** + * set_action + * @param mixed $action + * + * @return void + */ public function set_action($action) { $this->action = $action; } + /** + * add_object_manipulation + * @param int $filesize + * + * @return void + */ public function add_object_manipulation($filesize) { $this->totalfilesize += $filesize; $this->totalfilecount++; } + /** + * log_object_manipulation + * @return void + */ public function log_object_manipulation() { $duration = $this->timestart - $this->timeend; $totalfilesize = display_size($this->totalfilesize); @@ -67,6 +112,12 @@ public function log_object_manipulation() { mtrace($logstring); } + /** + * log_object_manipulation_query + * @param mixed $totalobjectsfound + * + * @return void + */ public function log_object_manipulation_query($totalobjectsfound) { $duration = $this->timeend - $this->timestart; $logstring = "Objectsfs $this->action manipulator took $duration seconds "; diff --git a/classes/local/object_manipulator/manipulator.php b/classes/local/object_manipulator/manipulator.php index a0cadaf8..f5108305 100644 --- a/classes/local/object_manipulator/manipulator.php +++ b/classes/local/object_manipulator/manipulator.php @@ -35,6 +35,9 @@ require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); +/** + * manipulator + */ abstract class manipulator implements object_manipulator { /** @@ -82,6 +85,7 @@ public function __construct(object_file_system $filesystem, stdClass $config, ag } /** + * execute * @param array $objectrecords * @return mixed|void * @throws dml_exception diff --git a/classes/local/object_manipulator/manipulator_builder.php b/classes/local/object_manipulator/manipulator_builder.php index 66e5ba41..ca7022c8 100644 --- a/classes/local/object_manipulator/manipulator_builder.php +++ b/classes/local/object_manipulator/manipulator_builder.php @@ -36,6 +36,9 @@ require_once(__DIR__ . '/../../../lib.php'); +/** + * manipulator_builder + */ class manipulator_builder { /** @var array $manipulators */ @@ -64,6 +67,7 @@ class manipulator_builder { private $candidates = []; /** + * execute * @param string $manipulator * @throws coding_exception * @throws moodle_exception @@ -83,6 +87,7 @@ public function execute($manipulator) { } /** + * execute_all * @throws coding_exception * @throws moodle_exception */ @@ -95,6 +100,7 @@ public function execute_all() { } /** + * build * @param string $manipulator * @throws moodle_exception */ diff --git a/classes/local/object_manipulator/object_manipulator.php b/classes/local/object_manipulator/object_manipulator.php index 039c0644..33d9d88a 100644 --- a/classes/local/object_manipulator/object_manipulator.php +++ b/classes/local/object_manipulator/object_manipulator.php @@ -31,12 +31,14 @@ interface object_manipulator { /** + * execute * @param array $objects * @return mixed */ public function execute(array $objects); /** + * manipulate_object * @param stdClass $objectrecord * @return int */ diff --git a/classes/local/object_manipulator/orphaner.php b/classes/local/object_manipulator/orphaner.php index 76b9bf0b..95463522 100644 --- a/classes/local/object_manipulator/orphaner.php +++ b/classes/local/object_manipulator/orphaner.php @@ -15,6 +15,8 @@ // along with Moodle. If not, see . /** + * Orphans records for files deleted + * * Orphans {tool_objectfs_objects} records for files that have been * deleted from the core {files} table. * @@ -29,6 +31,9 @@ use stdClass; +/** + * orphaner + */ class orphaner extends manipulator { /** diff --git a/classes/local/object_manipulator/puller.php b/classes/local/object_manipulator/puller.php index e22a1cf2..96dbeb11 100644 --- a/classes/local/object_manipulator/puller.php +++ b/classes/local/object_manipulator/puller.php @@ -27,9 +27,13 @@ use stdClass; +/** + * puller + */ class puller extends manipulator { /** + * manipulate_object * @param stdClass $objectrecord * @return int */ diff --git a/classes/local/object_manipulator/pusher.php b/classes/local/object_manipulator/pusher.php index fec59ef9..c63f6557 100644 --- a/classes/local/object_manipulator/pusher.php +++ b/classes/local/object_manipulator/pusher.php @@ -29,6 +29,9 @@ use tool_objectfs\local\store\object_file_system; use tool_objectfs\log\aggregate_logger; +/** + * pusher + */ class pusher extends manipulator { /** @@ -59,6 +62,7 @@ public function __construct(object_file_system $filesystem, stdClass $config, ag } /** + * manipulate_object * @param stdClass $objectrecord * @return int */ diff --git a/classes/local/object_manipulator/recoverer.php b/classes/local/object_manipulator/recoverer.php index e72926c9..7fa38d64 100644 --- a/classes/local/object_manipulator/recoverer.php +++ b/classes/local/object_manipulator/recoverer.php @@ -27,9 +27,13 @@ use stdClass; +/** + * recoverer + */ class recoverer extends manipulator { /** + * manipulate_object * @param stdClass $objectrecord * @return int */ diff --git a/classes/local/report/location_report_builder.php b/classes/local/report/location_report_builder.php index 2838e917..fe700198 100644 --- a/classes/local/report/location_report_builder.php +++ b/classes/local/report/location_report_builder.php @@ -28,9 +28,14 @@ use tool_objectfs\local\manager; use tool_objectfs\local\store\object_file_system; +/** + * location_report_builder + */ class location_report_builder extends objectfs_report_builder { /** + * build_report + * @param int $reportid * @return objectfs_report * @throws \dml_exception */ diff --git a/classes/local/report/log_size_report_builder.php b/classes/local/report/log_size_report_builder.php index ab703bfc..2295454f 100644 --- a/classes/local/report/log_size_report_builder.php +++ b/classes/local/report/log_size_report_builder.php @@ -25,8 +25,17 @@ namespace tool_objectfs\local\report; +/** + * log_size_report_builder + */ class log_size_report_builder extends objectfs_report_builder { + /** + * build_report + * @param int $reportid + * + * @return objectfs_report + */ public function build_report($reportid) { global $DB; @@ -49,6 +58,12 @@ public function build_report($reportid) { return $report; } + /** + * compress_small_log_sizes + * @param mixed $stats + * + * @return void + */ public function compress_small_log_sizes(&$stats) { $smallstats = new \stdClass(); $smallstats->datakey = 1; diff --git a/classes/local/report/mime_type_report_builder.php b/classes/local/report/mime_type_report_builder.php index 004b6565..964481ba 100644 --- a/classes/local/report/mime_type_report_builder.php +++ b/classes/local/report/mime_type_report_builder.php @@ -25,8 +25,17 @@ namespace tool_objectfs\local\report; +/** + * mime_type_report_builder + */ class mime_type_report_builder extends objectfs_report_builder { + /** + * build_report + * @param int $reportid + * + * @return objectfs_report + */ public function build_report($reportid) { global $DB; diff --git a/classes/local/report/object_status_history_table.php b/classes/local/report/object_status_history_table.php index 48d98a17..31191602 100644 --- a/classes/local/report/object_status_history_table.php +++ b/classes/local/report/object_status_history_table.php @@ -54,6 +54,9 @@ class object_status_history_table extends \table_sql { /** * Constructor for the file status history table. + * + * @param string $reporttype + * @param int $reportid */ public function __construct($reporttype, $reportid) { parent::__construct('statushistory'); diff --git a/classes/local/report/objectfs_report.php b/classes/local/report/objectfs_report.php index 20f6cb9e..86cd5e8b 100644 --- a/classes/local/report/objectfs_report.php +++ b/classes/local/report/objectfs_report.php @@ -25,19 +25,33 @@ namespace tool_objectfs\local\report; +/** + * objectfs_report + */ class objectfs_report implements \renderable { - /** @var string $reporttype */ + /** + * reporttype + * @var string + */ protected $reporttype = ''; - /** @var int $reportid */ + /** + * reportid + * @var int + */ protected $reportid = 0; - /** @var array $rows */ + /** + * rows + * @var array + */ protected $rows = []; /** + * construct * @param string $reporttype + * @param int $reportid */ public function __construct($reporttype, $reportid) { $this->reporttype = $reporttype; @@ -45,6 +59,7 @@ public function __construct($reporttype, $reportid) { } /** + * add_row * @param string $datakey * @param int $objectcount * @param int $objectsum @@ -58,6 +73,7 @@ public function add_row($datakey, $objectcount, $objectsum) { } /** + * add_rows * @param array $rows */ public function add_rows(array $rows) { @@ -67,6 +83,7 @@ public function add_rows(array $rows) { } /** + * get_rows * @return array */ public function get_rows() { @@ -74,6 +91,7 @@ public function get_rows() { } /** + * get_report_type * @return string */ public function get_report_type() { @@ -81,6 +99,7 @@ public function get_report_type() { } /** + * get_report_id * @return int */ public function get_report_id() { @@ -105,6 +124,10 @@ public function save_report_to_database() { } } + /** + * generate_status_report + * @return void + */ public static function generate_status_report() { global $DB; $reportid = $DB->insert_record('tool_objectfs_reports', (object)['reportdate' => time()]); @@ -135,6 +158,7 @@ public static function cleanup_reports() { } /** + * get_report_types * @return array */ public static function get_report_types() { diff --git a/classes/local/report/objectfs_report_builder.php b/classes/local/report/objectfs_report_builder.php index 1f1e467f..dd3046f1 100644 --- a/classes/local/report/objectfs_report_builder.php +++ b/classes/local/report/objectfs_report_builder.php @@ -25,7 +25,16 @@ namespace tool_objectfs\local\report; +/** + * objectfs_report_builder + */ abstract class objectfs_report_builder { + /** + * build_report + * @param int $reportid + * + * @return objectfs_report + */ abstract public function build_report($reportid); } diff --git a/classes/local/store/azure/client.php b/classes/local/store/azure/client.php index d4441249..2a6e3fa4 100644 --- a/classes/local/store/azure/client.php +++ b/classes/local/store/azure/client.php @@ -30,6 +30,9 @@ use tool_objectfs\local\store\azure\stream_wrapper; use tool_objectfs\local\store\object_client_base; +/** + * client + */ class client extends object_client_base { /** @var BlobRestProxy $client The Blob client. */ @@ -41,7 +44,7 @@ class client extends object_client_base { /** * The azure client constructor. * - * @param $config + * @param \stdclass $config */ public function __construct($config) { global $CFG; @@ -141,6 +144,10 @@ public function get_relative_path_from_fullpath($fullpath) { return $relativepath; } + /** + * get_seekable_stream_context + * @return resource + */ public function get_seekable_stream_context() { $context = stream_context_create(array( 'blob' => array( @@ -153,7 +160,7 @@ public function get_seekable_stream_context() { /** * Trim a leading '?' character from the sas token. * - * @param $sastoken + * @param string $sastoken * @return bool|string */ private function clean_sastoken($sastoken) { @@ -164,6 +171,12 @@ private function clean_sastoken($sastoken) { return $sastoken; } + /** + * get_md5_from_hash + * @param string $contenthash + * + * @return string + */ private function get_md5_from_hash($contenthash) { try { $key = $this->get_filepath_from_hash($contenthash); @@ -185,6 +198,13 @@ private function get_md5_from_hash($contenthash) { return $md5; } + /** + * verify_objectverify_object + * @param string $contenthash + * @param string $localpath + * + * @return bool + */ public function verify_object($contenthash, $localpath) { // For objects uploaded to S3 storage using the multipart upload, the etag will not be the objects MD5. // So we can't compare here to verify the object. @@ -196,12 +216,22 @@ public function verify_object($contenthash, $localpath) { return false; } + /** + * get_filepath_from_hash + * @param string $contenthash + * + * @return string + */ protected function get_filepath_from_hash($contenthash) { $l1 = $contenthash[0] . $contenthash[1]; $l2 = $contenthash[2] . $contenthash[3]; return "$l1/$l2/$contenthash"; } + /** + * test_connection + * @return stdClass + */ public function test_connection() { $connection = new \stdClass(); $connection->success = true; @@ -220,6 +250,12 @@ public function test_connection() { return $connection; } + /** + * test_permissions + * @param mixed $testdelete + * + * @return stdClass + */ public function test_permissions($testdelete) { $permissions = new \stdClass(); $permissions->success = true; @@ -268,6 +304,12 @@ public function test_permissions($testdelete) { return $permissions; } + /** + * get_exception_details + * @param \MicrosoftAzure\Storage\Common\Exceptions\ServiceException $exception + * + * @return string + */ protected function get_exception_details(\MicrosoftAzure\Storage\Common\Exceptions\ServiceException $exception) { $message = $exception->getErrorMessage(); @@ -298,7 +340,7 @@ protected function get_exception_details(\MicrosoftAzure\Storage\Common\Exceptio * Shared Access Signature. * * @param admin_settingpage $settings - * @param $config + * @param \stdClass $config * @return admin_settingpage */ public function define_client_section($settings, $config) { diff --git a/classes/local/store/azure/file_system.php b/classes/local/store/azure/file_system.php index fd5c203c..aa91803a 100644 --- a/classes/local/store/azure/file_system.php +++ b/classes/local/store/azure/file_system.php @@ -31,8 +31,17 @@ require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); +/** + * file_system + */ class file_system extends object_file_system { + /** + * initialise_external_client + * @param mixed $config + * + * @return client + */ protected function initialise_external_client($config) { $asclient = new client($config); return $asclient; diff --git a/classes/local/store/azure/stream_wrapper.php b/classes/local/store/azure/stream_wrapper.php index 00b67106..47a67a87 100644 --- a/classes/local/store/azure/stream_wrapper.php +++ b/classes/local/store/azure/stream_wrapper.php @@ -42,6 +42,9 @@ use MicrosoftAzure\Storage\Common\Exceptions\ServiceException; use Psr\Http\Message\StreamInterface; +/** + * stream_wrapper + */ class stream_wrapper { /** @var resource|null Stream context (this is set by PHP) */ @@ -85,16 +88,35 @@ public static function register(BlobRestProxy $client, $protocol = 'blob') { stream_context_set_default($default); } - public function stream_cast($cast_as) { + /** + * stream_cast + * @param mixed $cast_as + * + * @return boolean + */ + public function stream_cast($castas) { return false; } + /** + * stream_close + * @return void + */ public function stream_close() { $this->body = null; $this->hash = null; } - public function stream_open($path, $mode, $options, &$opened_path) { + /** + * stream_open + * @param mixed $path + * @param mixed $mode + * @param mixed $options + * @param mixed $opened_path + * + * @return bool + */ + public function stream_open($path, $mode, $options, &$openedpath) { $this->initProtocol($path); $this->params = $this->getContainerKey($path); $this->mode = rtrim($mode, 'bt'); @@ -107,17 +129,28 @@ public function stream_open($path, $mode, $options, &$opened_path) { return $this->boolCall(function() use ($path) { switch ($this->mode) { - case 'r': return $this->openReadStream(); - case 'a': return $this->openAppendStream(); - default: return $this->openWriteStream(); + case 'r': +return $this->openReadStream(); + case 'a': +return $this->openAppendStream(); + default: +return $this->openWriteStream(); } }); } + /** + * stream_eof + * @return bool + */ public function stream_eof() { return $this->body->eof(); } + /** + * stream_flush + * @return bool + */ public function stream_flush() { if ($this->mode == 'r') { return false; @@ -153,11 +186,24 @@ public function stream_flush() { }); } + /** + * stream_read + * @param int $count + * + * @return string + */ public function stream_read($count) { // If the file isn't readable, we need to return no content. Azure can emit XML here otherwise. return $this->readable ? $this->body->read($count) : ''; } + /** + * stream_seek + * @param int $offset + * @param int $whence + * + * @return bool + */ public function stream_seek($offset, $whence = SEEK_SET) { return !$this->body->isSeekable() ? false @@ -167,15 +213,30 @@ public function stream_seek($offset, $whence = SEEK_SET) { }); } + /** + * stream_tell + * @return bool + */ public function stream_tell() { - return $this->boolCall(function() { return $this->body->tell(); }); + return $this->boolCall(function() { return $this->body->tell(); + }); } + /** + * stream_write + * @param string $data + * + * @return int + */ public function stream_write($data) { hash_update($this->hash, $data); return $this->body->write($data); } + /** + * stream_stat + * @return array + */ public function stream_stat() { $stat = $this->getStatTemplate(); $stat[7] = $stat['size'] = $this->getSize(); @@ -185,9 +246,16 @@ public function stream_stat() { } /** + * url_stat + * * Provides information for is_dir, is_file, filesize, etc. Works on * buckets, keys, and prefixes. * @link http://www.php.net/manual/en/streamwrapper.url-stat.php + * + * @param string $path + * @param mixed $flags + * + * @return mixed */ public function url_stat($path, $flags) { $stat = $this->getStatTemplate(); @@ -219,17 +287,23 @@ public function url_stat($path, $flags) { /** * Parse the protocol out of the given path. * - * @param $path + * @param string $path */ - private function initProtocol($path) { + private function initprotocol($path) { $parts = explode('://', $path, 2); $this->protocol = $parts[0] ?: 'blob'; } - private function getContainerKey($path) { - // Remove the protocol + /** + * getContainerKey + * @param string $path + * + * @return array + */ + private function getcontainerkey($path) { + // Remove the protocol. $parts = explode('://', $path); - // Get the container, key + // Get the container, key. $parts = explode('/', $parts[1], 2); return [ @@ -242,6 +316,14 @@ private function getContainerKey($path) { * Validates the provided stream arguments for fopen and returns an array * of errors. */ + /** + * Validates the provided stream arguments for fopen and returns an array + * of errors. + * @param string $path + * @param string $mode + * + * @return [type] + */ private function validate($path, $mode) { $errors = []; @@ -255,8 +337,7 @@ private function validate($path, $mode) { . "Use one 'r', 'w', 'a', or 'x'."; } - // When using mode "x" validate if the file exists before attempting - // to read + // When using mode "x" validate if the file exists before attempting to read. if ($mode == 'x' && $this->getClient()->getBlobProperties( $this->getOption('Container'), @@ -288,8 +369,8 @@ private function validate($path, $mode) { * * @return array */ - private function getOptions($removeContextData = false) { - // Context is not set when doing things like stat + private function getoptions($removecontextdata = false) { + // Context is not set when doing things like stat. if ($this->context === null) { $options = []; } else { @@ -305,7 +386,7 @@ private function getOptions($removeContextData = false) { : []; $result = $this->params + $options + $default; - if ($removeContextData) { + if ($removecontextdata) { unset($result['client'], $result['seekable']); } @@ -319,7 +400,7 @@ private function getOptions($removeContextData = false) { * * @return mixed|null */ - private function getOption($name) { + private function getoption($name) { $options = $this->getOptions(); return isset($options[$name]) ? $options[$name] : null; @@ -331,7 +412,7 @@ private function getOption($name) { * @return BlobRestProxy * @throws \RuntimeException if no client has been configured */ - private function getClient() { + private function getclient() { if (!$client = $this->getOption('client')) { throw new \RuntimeException('No client in stream context'); } @@ -346,13 +427,17 @@ private function getClient() { * * @return array Hash of 'Container', 'Key', and custom params from the context */ - private function withPath($path) { + private function withpath($path) { $params = $this->getOptions(true); return $this->getContainerKey($path) + $params; } - private function openReadStream() { + /** + * openReadStream + * @return bool + */ + private function openreadstream() { $client = $this->getClient(); $params = $this->getOptions(true); @@ -365,7 +450,7 @@ private function openReadStream() { $this->body = $response->getBody(); } - // Wrap the body in a caching entity body if seeking is allowed + // Wrap the body in a caching entity body if seeking is allowed. if ($this->getOption('seekable') && !$this->body->isSeekable()) { $this->body = new CachingStream($this->body); } @@ -373,21 +458,29 @@ private function openReadStream() { return true; } - private function openWriteStream() { + /** + * openWriteStream + * @return bool + */ + private function openwritestream() { $this->body = new Stream(fopen('php://temp', 'r+')); return true; } - private function openAppendStream() { + /** + * openAppendStream + * @return mixed + */ + private function openappendstream() { try { - // Get the body of the object and seek to the end of the stream + // Get the body of the object and seek to the end of the stream. $client = $this->getClient(); $params = $this->getOptions(true); $this->body = $client->getBlob($params['Container'], $params['Key']); $this->body->seek(0, SEEK_END); return true; } catch (ServiceException $e) { - // The object does not exist, so use a simple write stream + // The object does not exist, so use a simple write stream. return $this->openWriteStream(); } } @@ -397,7 +490,7 @@ private function openAppendStream() { * * @return array */ - private function getStatTemplate() { + private function getstattemplate() { return [ 0 => 0, 'dev' => 0, 1 => 0, 'ino' => 0, @@ -424,7 +517,7 @@ private function getStatTemplate() { * * @return bool */ - private function boolCall(callable $fn, $flags = null) { + private function boolcall(callable $fn, $flags = null) { try { return $fn(); } catch (\Exception $e) { @@ -442,16 +535,16 @@ private function boolCall(callable $fn, $flags = null) { * @return bool Returns false * @throws \RuntimeException if throw_errors is true */ - private function triggerError($errors, $flags = null) { - // This is triggered with things like file_exists() + private function triggererror($errors, $flags = null) { + // This is triggered with things like file_exists(). if ($flags & STREAM_URL_STAT_QUIET) { return $flags & STREAM_URL_STAT_LINK - // This is triggered for things like is_link() + // This is triggered for things like is_link(). ? $this->getStatTemplate() : false; } - // This is triggered when doing things like lstat() or stat() + // This is triggered when doing things like lstat() or stat(). trigger_error(implode("\n", (array) $errors), E_USER_WARNING); return false; @@ -462,7 +555,7 @@ private function triggerError($errors, $flags = null) { * * @return int|null */ - private function getSize() { + private function getsize() { $size = $this->body->getSize(); return $size !== null ? $size : $this->size; diff --git a/classes/local/store/digitalocean/client.php b/classes/local/store/digitalocean/client.php index 0ee9a2af..644043fe 100644 --- a/classes/local/store/digitalocean/client.php +++ b/classes/local/store/digitalocean/client.php @@ -19,6 +19,7 @@ * * @package tool_objectfs * @author Brian Yanosik + * @copyright Brian Yanosik * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ @@ -26,8 +27,16 @@ use tool_objectfs\local\store\s3\client as s3_client; +/** + * client + */ class client extends s3_client { + /** + * construct + * @param \stdClass $config + * @return void + */ public function __construct($config) { global $CFG; $this->autoloader = $CFG->dirroot . '/local/aws/sdk/aws-autoloader.php'; @@ -56,6 +65,12 @@ protected function is_configured($config) { return true; } + /** + * set_client + * @param \stdClass $config + * + * @return void + */ public function set_client($config) { if (!$this->is_configured($config)) { $this->client = null; @@ -71,8 +86,9 @@ public function set_client($config) { } /** + * define_client_section * @param admin_settingpage $settings - * @param $config + * @param \stdClass $config * @return admin_settingpage */ public function define_client_section($settings, $config) { diff --git a/classes/local/store/digitalocean/file_system.php b/classes/local/store/digitalocean/file_system.php index 0595ff9d..5b37eaaa 100644 --- a/classes/local/store/digitalocean/file_system.php +++ b/classes/local/store/digitalocean/file_system.php @@ -22,6 +22,7 @@ * * @package tool_objectfs * @author Brian Yanosik + * @copyright Brian Yanosik * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ @@ -33,8 +34,17 @@ require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); +/** + * [Description file_system] + */ class file_system extends s3_file_system { + /** + * initialise_external_client + * @param \stdClass $config + * + * @return client + */ protected function initialise_external_client($config) { $doclient = new client($config); diff --git a/classes/local/store/object_client.php b/classes/local/store/object_client.php index 44bdc306..3c32f1ea 100644 --- a/classes/local/store/object_client.php +++ b/classes/local/store/object_client.php @@ -26,21 +26,117 @@ namespace tool_objectfs\local\store; interface object_client { + + /** + * construct + * @param \stdClass $config + */ public function __construct($config); + + /** + * register_stream_wrapper + * @return mixed + */ public function register_stream_wrapper(); + + /** + * get_fullpath_from_hash + * @param string $contenthash + * + * @return string + */ public function get_fullpath_from_hash($contenthash); + + /** + * delete_file + * @param string $fullpath + * + * @return mixed + */ public function delete_file($fullpath); + + /** + * rename_file + * @param string $currentpath + * @param string $destinationpath + * + * @return mixed + */ public function rename_file($currentpath, $destinationpath); + + /** + * get_seekable_stream_context + * @return mixed + */ public function get_seekable_stream_context(); + + /** + * get_availability + * @return mixed + */ public function get_availability(); + + /** + * get_maximum_upload_size + * @return mixed + */ public function get_maximum_upload_size(); + + /** + * verify_object + * @param string $contenthash + * @param string $localpath + * + * @return mixed + */ public function verify_object($contenthash, $localpath); + + /** + * generate_presigned_url + * @param string $contenthash + * @param array $headers + * + * @return mixed + */ public function generate_presigned_url($contenthash, $headers = array()); + + /** + * support_presigned_urls + * @return mixed + */ public function support_presigned_urls(); + + /** + * test_connection + * @return mixed + */ public function test_connection(); + + /** + * test_permissions + * @param mixed $testdelete + * + * @return mixed + */ public function test_permissions($testdelete); + + /** + * proxy_range_request + * @param \stored_file $file + * @param mixed $ranges + * + * @return mixed + */ public function proxy_range_request(\stored_file $file, $ranges); + + /** + * test_range_request + * @param mixed $filesystem + * + * @return mixed + */ public function test_range_request($filesystem); + } diff --git a/classes/local/store/object_client_base.php b/classes/local/store/object_client_base.php index cda37840..358a0866 100644 --- a/classes/local/store/object_client_base.php +++ b/classes/local/store/object_client_base.php @@ -25,12 +25,30 @@ namespace tool_objectfs\local\store; +/** + * [Description object_client_base] + */ abstract class object_client_base implements object_client { + /** + * @var string + */ protected $autoloader; + /** + * @var mixed + */ protected $expirationtime; + /** + * @var bool + */ protected $testdelete = true; + /** + * @var int + */ public $presignedminfilesize; + /** + * @var mixed + */ public $enablepresignedurls; /** @var int $maxupload Maximum allowed file size that can be uploaded. */ @@ -39,6 +57,10 @@ abstract class object_client_base implements object_client { /** @var object $config Client config. */ protected $config; + /** + * construct + * @param \stdClass $config + */ public function __construct($config) { } @@ -56,6 +78,10 @@ public function get_availability() { } } + /** + * register_stream_wrapper + * @return void + */ public function register_stream_wrapper() { } diff --git a/classes/local/store/object_file_system.php b/classes/local/store/object_file_system.php index 10692f1a..f66b4c4a 100644 --- a/classes/local/store/object_file_system.php +++ b/classes/local/store/object_file_system.php @@ -45,13 +45,32 @@ require_once($CFG->libdir . '/filestorage/file_system_filedir.php'); require_once($CFG->libdir . '/filestorage/file_storage.php'); +/** + * [Description object_file_system] + */ abstract class object_file_system extends \file_system_filedir { + /** + * @var mixed + */ public $externalclient; + /** + * @var mixed + */ private $preferexternal; + /** + * @var mixed + */ private $deleteexternally; + /** + * @var mixed + */ private $logger; + /** + * construct + * @return void + */ public function __construct() { global $CFG; parent::__construct(); // Setup filedir. @@ -72,6 +91,12 @@ public function __construct() { } } + /** + * set_logger + * @param \tool_objectfs\log\objectfs_logger $logger + * + * @return void + */ public function set_logger(\tool_objectfs\log\objectfs_logger $logger) { $this->logger = $logger; } @@ -94,6 +119,12 @@ public function get_external_client() { return $this->externalclient; } + /** + * initialise_external_client + * @param mixed $config + * + * @return mixed + */ abstract protected function initialise_external_client($config); /** @@ -133,10 +164,22 @@ protected function get_local_path_from_hash($contenthash, $fetchifnotfound = fal return $path; } + /** + * get_remote_path_from_storedfile + * @param \stored_file $file + * + * @return string + */ public function get_remote_path_from_storedfile(\stored_file $file) { return $this->get_remote_path_from_hash($file->get_contenthash()); } + /** + * get_remote_path_from_hash + * @param mixed $contenthash + * + * @return string + */ protected function get_remote_path_from_hash($contenthash) { if ($this->preferexternal) { $location = $this->get_object_location_from_hash($contenthash); @@ -155,14 +198,32 @@ protected function get_remote_path_from_hash($contenthash) { return $path; } + /** + * get_external_path_from_hash + * @param mixed $contenthash + * + * @return string + */ protected function get_external_path_from_hash($contenthash) { return $this->externalclient->get_fullpath_from_hash($contenthash); } + /** + * get_external_path_from_storedfile + * @param \stored_file $file + * + * @return string + */ protected function get_external_path_from_storedfile(\stored_file $file) { return $this->get_external_path_from_hash($file->get_contenthash()); } + /** + * is_file_readable_externally_by_storedfile + * @param stored_file $file + * + * @return bool + */ public function is_file_readable_externally_by_storedfile(stored_file $file) { if (!$file->get_filesize()) { // Files with empty size are either directories or empty. @@ -178,6 +239,12 @@ public function is_file_readable_externally_by_storedfile(stored_file $file) { return false; } + /** + * is_file_readable_externally_by_hash + * @param mixed $contenthash + * + * @return bool + */ public function is_file_readable_externally_by_hash($contenthash) { if ($contenthash === sha1('')) { // Files with empty size are either directories or empty. @@ -191,6 +258,12 @@ public function is_file_readable_externally_by_hash($contenthash) { return is_readable($path); } + /** + * get_object_location_from_hash + * @param mixed $contenthash + * + * @return int + */ public function get_object_location_from_hash($contenthash) { $localreadable = $this->is_file_readable_locally_by_hash($contenthash); $externalreadable = $this->is_file_readable_externally_by_hash($contenthash); @@ -208,7 +281,13 @@ public function get_object_location_from_hash($contenthash) { } } - // Acquire the object lock any time you are moving an object between locations. + /** + * Acquire the object lock any time you are moving an object between locations. + * @param mixed $contenthash + * @param int $timeout + * + * @return \core\lock\lock|boolean + */ public function acquire_object_lock($contenthash, $timeout = 0) { $resource = "tool_objectfs: $contenthash"; $lockfactory = \core\lock\lock_config::get_lock_factory('tool_objectfs_object'); @@ -220,6 +299,13 @@ public function acquire_object_lock($contenthash, $timeout = 0) { return $lock; } + /** + * copy_object_from_external_to_local_by_hash + * @param mixed $contenthash + * @param int $objectsize + * + * @return int + */ public function copy_object_from_external_to_local_by_hash($contenthash, $objectsize = 0) { $initiallocation = $this->get_object_location_from_hash($contenthash); $finallocation = $initiallocation; @@ -253,6 +339,13 @@ public function copy_object_from_external_to_local_by_hash($contenthash, $object return $finallocation; } + /** + * copy_object_from_local_to_external_by_hash + * @param mixed $contenthash + * @param int $objectsize + * + * @return int + */ public function copy_object_from_local_to_external_by_hash($contenthash, $objectsize = 0) { $initiallocation = $this->get_object_location_from_hash($contenthash); @@ -275,12 +368,25 @@ public function copy_object_from_local_to_external_by_hash($contenthash, $object return $finallocation; } + /** + * verify_external_object_from_hash + * @param mixed $contenthash + * + * @return mixed + */ public function verify_external_object_from_hash($contenthash) { $localpath = $this->get_local_path_from_hash($contenthash); $objectisvalid = $this->externalclient->verify_object($contenthash, $localpath); return $objectisvalid; } + /** + * delete_object_from_local_by_hash + * @param mixed $contenthash + * @param int $objectsize + * + * @return int + */ public function delete_object_from_local_by_hash($contenthash, $objectsize = 0) { $initiallocation = $this->get_object_location_from_hash($contenthash); $finallocation = $initiallocation; @@ -622,6 +728,9 @@ public function copy_file_from_hash_to_path($contenthash, $destinationpath) { * Deletes external file depending on deleteexternal settings. * * @param string $contenthash file to be moved + * @param bool $force + * + * @return void */ public function delete_external_file_from_hash($contenthash, $force = false) { if ($force || (!empty($this->deleteexternally) && $this->deleteexternally == TOOL_OBJECTFS_DELETE_EXTERNAL_FULL)) { @@ -692,7 +801,7 @@ public function get_client_availability() { /** * Delete file with external client. * - * @path path to file to be deleted. + * @param string $path path to file to be deleted. * @return bool. */ public function delete_client_file($path) { @@ -784,6 +893,10 @@ public function supports_presigned_urls() { return false; } + /** + * presigned_url_configured + * @return mixed + */ public function presigned_url_configured() { return $this->externalclient->support_presigned_urls() && $this->externalclient->enablepresignedurls; @@ -877,6 +990,7 @@ public function get_filedir_count() { } /** + * exec_command * @param string $command * @return int */ @@ -1024,8 +1138,8 @@ public function add_file_from_string($content) { /** * Update file remote location. * - * @param array (contenthash, filesize, newfile) - * @return array (contenthash, filesize, newfile) + * @param array $result [contenthash, filesize, newfile] + * @return array [contenthash, filesize, newfile] */ private function update_object(array $result): array { // Rather than getting its exact location we just set it to local. diff --git a/classes/local/store/s3/client.php b/classes/local/store/s3/client.php index 12ae1de0..f029acf7 100644 --- a/classes/local/store/s3/client.php +++ b/classes/local/store/s3/client.php @@ -35,6 +35,9 @@ define('AWS_CAN_WRITE_OBJECT', 1); define('AWS_CAN_DELETE_OBJECT', 2); +/** + * [Description client] + */ class client extends object_client_base { /** @@ -44,13 +47,26 @@ class client extends object_client_base { */ const MAX_TEMP_LIMIT = 2097152; + /** + * @var mixed + */ protected $client; + /** + * @var mixed + */ protected $bucket; + /** + * @var mixed + */ private $signingmethod; /** @var string Prefix for bucket keys */ protected $bucketkeyprefix; + /** + * construct + * @param mixed $config + */ public function __construct($config) { global $CFG; $this->autoloader = $CFG->dirroot . '/local/aws/sdk/aws-autoloader.php'; @@ -73,10 +89,18 @@ public function __construct($config) { } } + /** + * sleep + * @return array + */ public function __sleep() { return array('bucket'); } + /** + * wakeup + * @return void + */ public function __wakeup() { // We dont want to store credentials in the client itself as // it will be serialised, so re-retrive them now. @@ -121,6 +145,7 @@ protected function is_configured($config) { * Set the client. * * @param \stdClass $config Client config. + * @return void */ public function set_client($config) { if (!$this->is_configured($config)) { @@ -151,7 +176,7 @@ public function set_client($config) { /** * Registers 's3://bucket' as a prefix for file actions. - * + * @return void */ public function register_stream_wrapper() { if ($this->get_availability() && $this->is_functional()) { @@ -161,6 +186,12 @@ public function register_stream_wrapper() { } } + /** + * get_md5_from_hash + * @param string $contenthash + * + * @return string|bool + */ private function get_md5_from_hash($contenthash) { if (!$this->is_functional()) { return false; @@ -180,6 +211,13 @@ private function get_md5_from_hash($contenthash) { return $md5; } + /** + * verify_object + * @param string $contenthash + * @param string $localpath + * + * @return bool + */ public function verify_object($contenthash, $localpath) { // For objects uploaded to S3 storage using the multipart upload, the etag will not be the objects MD5. // So we can't compare here to verify the object. @@ -205,7 +243,8 @@ public function get_fullpath_from_hash($contenthash) { /** * Deletes a file in S3 storage. * - * @path string full path to S3 file. + * @param string $fullpath full path to S3 file. + * @return void */ public function delete_file($fullpath) { unlink($fullpath); @@ -216,6 +255,7 @@ public function delete_file($fullpath) { * * @param string $currentpath current full path to S3 file. * @param string $destinationpath destination path. + * @return void */ public function rename_file($currentpath, $destinationpath) { rename($currentpath, $destinationpath); @@ -225,7 +265,7 @@ public function rename_file($currentpath, $destinationpath) { * S3 file streams require a seekable context to be supplied * if they are to be seekable. * - * @return void + * @return mixed */ public function get_seekable_stream_context() { $context = stream_context_create(array( @@ -236,6 +276,12 @@ public function get_seekable_stream_context() { return $context; } + /** + * get_filepath_from_hash + * @param string $contenthash + * + * @return string + */ protected function get_filepath_from_hash($contenthash) { $l1 = $contenthash[0] . $contenthash[1]; $l2 = $contenthash[2] . $contenthash[3]; @@ -281,6 +327,7 @@ public function test_connection() { * There is no check connection in the AWS API. * We use list buckets instead and check the bucket is in the list. * + * @param bool $testdelete * @return object * @throws \coding_exception */ @@ -322,7 +369,10 @@ public function test_permissions($testdelete) { if ($testdelete) { try { - $result = $this->client->deleteObject(array('Bucket' => $this->bucket, 'Key' => $this->bucketkeyprefix . 'permissions_check_file')); + $result = $this->client->deleteObject([ + 'Bucket' => $this->bucket, + 'Key' => $this->bucketkeyprefix . 'permissions_check_file' + ]); $permissions->messages[get_string('settings:deletesuccess', 'tool_objectfs')] = 'warning'; $permissions->success = false; } catch (\Aws\S3\Exception\S3Exception $e) { @@ -343,6 +393,12 @@ public function test_permissions($testdelete) { return $permissions; } + /** + * get_exception_details + * @param \Exception $exception + * + * @return string + */ protected function get_exception_details($exception) { $message = $exception->getMessage(); @@ -449,7 +505,11 @@ public function upload_to_s3($localpath, $contenthash) { try { $externalpath = $this->get_filepath_from_hash($contenthash); - $uploader = new \Aws\S3\ObjectUploader($this->client, $this->bucket, $this->bucketkeyprefix . $externalpath, $filehandle); + $uploader = new \Aws\S3\ObjectUploader( + $this->client, $this->bucket, + $this->bucketkeyprefix . $externalpath, + $filehandle + ); $uploader->upload(); fclose($filehandle); } catch (\Aws\Exception\MultipartUploadException $e) { @@ -487,6 +547,7 @@ public function generate_presigned_url($contenthash, $headers = array()) { } /** + * generate_presigned_url_s3 * @param string $contenthash * @param array $headers * @param bool $nicefilename @@ -524,6 +585,7 @@ private function generate_presigned_url_s3($contenthash, array $headers = [], $n } /** + * generate_presigned_url_cloudfront * @param string $contenthash * @param array $headers * @param bool $nicefilename @@ -580,7 +642,8 @@ private function generate_presigned_url_cloudfront($contenthash, array $headers } /** - * @param $headers + * get_nice_filename + * @param array $headers * @return array */ private function get_nice_filename($headers) { diff --git a/classes/local/store/s3/file_system.php b/classes/local/store/s3/file_system.php index d4689de0..dd911092 100644 --- a/classes/local/store/s3/file_system.php +++ b/classes/local/store/s3/file_system.php @@ -35,8 +35,17 @@ require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); +/** + * [Description file_system] + */ class file_system extends object_file_system { + /** + * initialise_external_client + * @param \stdClass $config + * + * @return client + */ protected function initialise_external_client($config) { $s3client = new client($config); @@ -44,7 +53,10 @@ protected function initialise_external_client($config) { } /** - * @inheritdoc + * readfile + * @param \stored_file $file + * @return void + * @throws \file_exception */ public function readfile(\stored_file $file) { $path = $this->get_remote_path_from_storedfile($file); @@ -66,7 +78,10 @@ public function readfile(\stored_file $file) { } /** - * @inheritdoc + * copy_from_local_to_external + * @param mixed $contenthash + * + * @return bool */ public function copy_from_local_to_external($contenthash) { $localpath = $this->get_local_path_from_hash($contenthash); @@ -83,7 +98,8 @@ public function copy_from_local_to_external($contenthash) { } /** - * @inheritdoc + * supports_presigned_urls + * @return bool */ public function supports_presigned_urls() { return true; diff --git a/classes/local/store/signed_url.php b/classes/local/store/signed_url.php index d00616f5..80f41ce8 100644 --- a/classes/local/store/signed_url.php +++ b/classes/local/store/signed_url.php @@ -17,6 +17,8 @@ namespace tool_objectfs\local\store; /** + * A signed URL for direct downloads + * * A signed URL which can be used by a user to directly download a file from object store, rather * than from the Moodle server. * @@ -36,6 +38,7 @@ class signed_url { public $expiresat; /** + * construct * @param \moodle_url $url URL to redirect to * @param int $expiresat Expiry timestamp (Unix epoch) after which this URL will stop working */ diff --git a/classes/local/store/swift/client.php b/classes/local/store/swift/client.php index e95a0fe0..1d520fba 100644 --- a/classes/local/store/swift/client.php +++ b/classes/local/store/swift/client.php @@ -28,6 +28,9 @@ use tool_objectfs\local\store\object_client_base; use tool_objectfs\local\manager; +/** + * [Description client] + */ class client extends object_client_base { /** @var string $containername The current container. */ @@ -36,7 +39,7 @@ class client extends object_client_base { /** * The swift client constructor. * - * @param $config + * @param \stdClass $config */ public function __construct($config) { global $CFG; @@ -53,6 +56,10 @@ public function __construct($config) { } } + /** + * get_endpoint + * @return array + */ private function get_endpoint() { $endpoint = [ @@ -67,7 +74,12 @@ private function get_endpoint() { ]; if (!isset($this->config->openstack_authtoken['expires_at']) - || (new \DateTimeImmutable($this->config->openstack_authtoken['expires_at'])) < ( (new \DateTimeImmutable('now'))->add(new \DateInterval('PT1H')))) { + || ( + new \DateTimeImmutable($this->config->openstack_authtoken['expires_at'])) + < + ( (new \DateTimeImmutable('now'))->add(new \DateInterval('PT1H')) + ) + ) { $lockfactory = \core\lock\lock_config::get_lock_factory('tool_objectfs_swift'); @@ -86,13 +98,23 @@ private function get_endpoint() { } // Use the token if it's valid, otherwise clients will need to use username/password auth. - if (isset($this->config->openstack_authtoken['expires_at']) && new \DateTimeImmutable($this->config->openstack_authtoken['expires_at']) > new \DateTimeImmutable('now')) { + if ( + isset($this->config->openstack_authtoken['expires_at']) + && + new \DateTimeImmutable($this->config->openstack_authtoken['expires_at']) + > + new \DateTimeImmutable('now') + ) { $endpoint['cachedToken'] = $this->config->openstack_authtoken; } return $endpoint; } + /** + * get_container + * @return mixed + */ public function get_container() { if (empty($this->config->openstack_authurl)) { @@ -118,7 +140,7 @@ public function register_stream_wrapper() { return; } - stream_wrapper_register('swift', "tool_objectfs\local\store\swift\stream_wrapper") or die("cant create wrapper"); + stream_wrapper_register('swift', "tool_objectfs\local\store\swift\stream_wrapper") || die("cant create wrapper"); \tool_objectfs\local\store\swift\stream_wrapper::set_default_context($this->get_seekable_stream_context()); $bootstraped = true; @@ -129,12 +151,22 @@ public function register_stream_wrapper() { } + /** + * get_fullpath_from_hash + * @param mixed $contenthash + * + * @return string + */ public function get_fullpath_from_hash($contenthash) { $filepath = $this->get_filepath_from_hash($contenthash); return "swift://$this->containername/$filepath"; } + /** + * get_seekable_stream_context + * @return resource + */ public function get_seekable_stream_context() { $this->get_endpoint(); @@ -153,6 +185,12 @@ public function get_seekable_stream_context() { } + /** + * get_md5_from_hash + * @param mixed $contenthash + * + * @return mixed + */ private function get_md5_from_hash($contenthash) { try { @@ -169,6 +207,13 @@ private function get_md5_from_hash($contenthash) { return $obj->hash; } + /** + * verify_object + * @param string $contenthash + * @param string $localpath + * + * @return bool + */ public function verify_object($contenthash, $localpath) { // For objects uploaded to S3 storage using the multipart upload, the etag will not be the objects MD5. // So we can't compare here to verify the object. @@ -180,6 +225,12 @@ public function verify_object($contenthash, $localpath) { return false; } + /** + * get_filepath_from_hash + * @param string $contenthash + * + * @return string + */ protected function get_filepath_from_hash($contenthash) { $l1 = $contenthash[0] . $contenthash[1]; $l2 = $contenthash[2] . $contenthash[3]; @@ -188,6 +239,10 @@ protected function get_filepath_from_hash($contenthash) { } + /** + * test_connection + * @return \stdClass + */ public function test_connection() { $connection = new \stdClass(); @@ -224,6 +279,12 @@ public function test_connection() { return $connection; } + /** + * test_permissions + * @param mixed $testdelete + * + * @return \stdClass + */ public function test_permissions($testdelete) { $permissions = new \stdClass(); $permissions->success = true; @@ -270,6 +331,12 @@ public function test_permissions($testdelete) { return $permissions; } + /** + * get_exception_details + * @param \OpenStack\Common\Error\BadResponseError $e + * + * @return string + */ protected function get_exception_details(\OpenStack\Common\Error\BadResponseError $e) { $message = $e->getResponse()->getReasonPhrase(); @@ -285,9 +352,9 @@ protected function get_exception_details(\OpenStack\Common\Error\BadResponseErro /** * swift settings form with the following elements: * - * @param admin_settingpage $settings - * @param $config - * @return admin_settingpage + * @param \admin_settingpage $settings + * @param \stdClass $config + * @return \admin_settingpage */ public function define_client_section($settings, $config) { @@ -329,7 +396,7 @@ public function define_client_section($settings, $config) { /** * Return the error code * - * @param $e The exception that contains the XML body. + * @param \Exception $e The exception that contains the XML body. * @return int The error code. */ private function get_error_code($e) { diff --git a/classes/local/store/swift/file_system.php b/classes/local/store/swift/file_system.php index 446cbdb6..5ac856ca 100644 --- a/classes/local/store/swift/file_system.php +++ b/classes/local/store/swift/file_system.php @@ -31,8 +31,17 @@ require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); +/** + * [Description file_system] + */ class file_system extends object_file_system { + /** + * initialise_external_client + * @param \stdClass $config + * + * @return client + */ protected function initialise_external_client($config) { $client = new client($config); return $client; diff --git a/classes/local/store/swift/stream_wrapper.php b/classes/local/store/swift/stream_wrapper.php index aa07f975..deee7e66 100644 --- a/classes/local/store/swift/stream_wrapper.php +++ b/classes/local/store/swift/stream_wrapper.php @@ -168,9 +168,7 @@ class stream_wrapper { private static $defaultcontext; - // - // Stream API functions - // + // Stream API functions. /** * Close directory @@ -230,10 +228,18 @@ public function dir_readdir() { return $item->name; } + /** + * stream_close + * @return bool + */ public function stream_close() { return $this->push_object(); } + /** + * stream_eof + * @return bool + */ public function stream_eof() { return $this->objstream->eof(); } @@ -427,10 +433,10 @@ public function unlink($path) { /** * Rename object * - * @param string $path + * @param string $currentpath + * @param string $destinationpath * @return boolean */ - public function rename($currentpath, $destinationpath) { $currenturl = $this->parse_url($currentpath); $destinationurl = $this->parse_url($destinationpath); @@ -488,10 +494,7 @@ public function url_stat($path, $flags) { return $this->generate_stat($object, $container, $object->contentLength); } - - // - // Non-stream API functions - // + // Non-stream API functions. /** * Set context for functions that don't accept a context. e.g. stat() @@ -633,7 +636,7 @@ private function set_mode($mode) { $this->isbinary = strpos($mode, 'b') !== false; $this->istext = strpos($mode, 't') !== false; - // rewrite mode to remove b or t: + // Rewrite mode to remove b or t:. $mode = preg_replace('/[bt]?/', '', $mode); switch ($mode) { diff --git a/classes/local/table/files_table.php b/classes/local/table/files_table.php index 82852849..eeb7ed70 100644 --- a/classes/local/table/files_table.php +++ b/classes/local/table/files_table.php @@ -29,6 +29,9 @@ require_once($CFG->libdir . '/tablelib.php'); +/** + * [Description files_table] + */ class files_table extends \table_sql { /** @@ -62,18 +65,42 @@ public function __construct($uniqueid, $objectlocation) { $this->downloadable = true; } + /** + * col_id + * @param \stdClass $row + * + * @return int + */ public function col_id(\stdClass $row) { return $row->id; } + /** + * col_contextid + * @param \stdClass $row + * + * @return int + */ public function col_contextid(\stdClass $row) { return $row->contextid; } + /** + * col_contenthash + * @param \stdClass $row + * + * @return string + */ public function col_contenthash(\stdClass $row) { return $row->contenthash; } + /** + * col_localpath + * @param \stdClass $row + * + * @return string + */ public function col_localpath(\stdClass $row) { $l1 = $row->contenthash[0] . $row->contenthash[1]; $l2 = $row->contenthash[2] . $row->contenthash[3]; @@ -81,34 +108,82 @@ public function col_localpath(\stdClass $row) { return "$l1/$l2"; } + /** + * col_component + * @param \stdClass $row + * + * @return mixed + */ public function col_component(\stdClass $row) { return $row->component; } + /** + * col_filearea + * @param \stdClass $row + * + * @return mixed + */ public function col_filearea(\stdClass $row) { return $row->filearea; } + /** + * col_filename + * @param \stdClass $row + * + * @return mixed + */ public function col_filename(\stdClass $row) { return $row->filename; } + /** + * col_filepath + * @param \stdClass $row + * + * @return string + */ public function col_filepath(\stdClass $row) { return $row->filepath; } + /** + * col_mimetype + * @param \stdClass $row + * + * @return string + */ public function col_mimetype(\stdClass $row) { return $row->mimetype; } + /** + * col_filesize + * @param \stdClass $row + * + * @return mixed + */ public function col_filesize(\stdClass $row) { return display_size($row->filesize); } + /** + * col_timecreated + * @param \stdClass $row + * + * @return int + */ public function col_timecreated(\stdClass $row) { return userdate($row->timecreated); } + /** + * col_link + * @param \stdClass $row + * + * @return string + */ public function col_link(\stdClass $row) { global $DB; diff --git a/classes/log/aggregate_logger.php b/classes/log/aggregate_logger.php index bdd23d78..c26029a5 100644 --- a/classes/log/aggregate_logger.php +++ b/classes/log/aggregate_logger.php @@ -31,13 +31,37 @@ require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); +/** + * [Description aggregate_logger] + */ class aggregate_logger extends objectfs_logger { - private $readstatistics; // 1d array of objectfs_statistics. - private $movestatistics; // 2d array of objecfs_statistics that is lazily setup. + /** + * 1d array of objectfs_statistics. + * @var array + */ + private $readstatistics; + + /** + * 2d array of objecfs_statistics that is lazily setup. + * @var array + */ + private $movestatistics; + + /** + * @var mixed + */ private $movement; + + /** + * @var array + */ private $querystatistics; + /** + * construct + * @return void + */ public function __construct() { parent::__construct(); $this->movestatistics = array( @@ -50,6 +74,14 @@ public function __construct() { $this->querystatistics = array(); } + /** + * log_object_read + * @param string $readname + * @param string $objectpath + * @param int $objectsize + * + * @return void + */ public function log_object_read($readname, $objectpath, $objectsize = 0) { if (array_key_exists($readname, $this->readstatistics)) { $readstat = $this->readstatistics[$readname]; @@ -61,6 +93,16 @@ public function log_object_read($readname, $objectpath, $objectsize = 0) { $this->readstatistics[$readname] = $readstat; } + /** + * log_object_move + * @param mixed $movename + * @param mixed $initallocation + * @param mixed $finallocation + * @param mixed $objecthash + * @param int $objectsize + * + * @return void + */ public function log_object_move($movename, $initallocation, $finallocation, $objecthash, $objectsize = 0) { if (!$this->movement) { $this->movement = $movename; @@ -76,6 +118,10 @@ public function log_object_move($movename, $initallocation, $finallocation, $obj $this->movestatistics[$initallocation][$finallocation] = $movestat; } + /** + * output_move_statistics + * @return void + */ public function output_move_statistics() { $totaltime = $this->get_timing(); mtrace("$this->movement. Total time taken: $totaltime seconds. Location change summary:"); @@ -86,6 +132,14 @@ public function output_move_statistics() { } } + /** + * output_move_statistic + * @param mixed $movestatistic + * @param string $initiallocation + * @param string $finallocation + * + * @return void + */ protected function output_move_statistic($movestatistic, $initiallocation, $finallocation) { $key = $movestatistic->get_key(); $objectcount = $movestatistic->get_objectcount(); @@ -96,6 +150,12 @@ protected function output_move_statistic($movestatistic, $initiallocation, $fina mtrace("$initiallocation -> $finallocation. Objects moved: $objectcount. Total size: $objectsum. "); } + /** + * location_to_string + * @param int $location + * + * @return string + */ public function location_to_string($location) { switch ($location) { case OBJECT_LOCATION_ERROR: @@ -113,6 +173,14 @@ public function location_to_string($location) { } } + /** + * log_object_query + * @param mixed $queryname + * @param int $objectcount + * @param int $objectsum + * + * @return [type] + */ public function log_object_query($queryname, $objectcount, $objectsum = 0) { if (array_key_exists($queryname, $this->querystatistics)) { $querystat = $this->querystatistics[$queryname]; diff --git a/classes/log/null_logger.php b/classes/log/null_logger.php index 4e1606a2..f29dbb47 100644 --- a/classes/log/null_logger.php +++ b/classes/log/null_logger.php @@ -29,16 +29,45 @@ require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); +/** + * [Description null_logger] + */ class null_logger extends objectfs_logger { + /** + * log_object_read + * @param mixed $readname + * @param mixed $objectpath + * @param int $objectsize + * + * @return void + */ public function log_object_read($readname, $objectpath, $objectsize = 0) { return; } + /** + * log_object_move + * @param mixed $movename + * @param mixed $initallocation + * @param mixed $finallocation + * @param mixed $objecthash + * @param int $objectsize + * + * @return void + */ public function log_object_move($movename, $initallocation, $finallocation, $objecthash, $objectsize = 0) { return; } + /** + * log_object_query + * @param mixed $queryname + * @param mixed $objectcount + * @param int $objectsum + * + * @return void + */ public function log_object_query($queryname, $objectcount, $objectsum = 0) { return; } diff --git a/classes/log/objectfs_logger.php b/classes/log/objectfs_logger.php index a0a7c427..6dfe6f38 100644 --- a/classes/log/objectfs_logger.php +++ b/classes/log/objectfs_logger.php @@ -29,40 +29,104 @@ require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); +/** + * [Description objectfs_logger] + */ abstract class objectfs_logger { + /** + * @var float + */ protected $timestart; + /** + * @var float + */ protected $timeend; + /** + * construct + */ public function __construct() { $this->timestart = 0; $this->timeend = 0; } + /** + * start_timing + * @return float + */ public function start_timing() { $this->timestart = microtime(true); return $this->timestart; } + /** + * end_timing + * @return float + */ public function end_timing() { $this->timeend = microtime(true); return $this->timeend; } + /** + * get_timing + * @return float + */ protected function get_timing() { return $this->timeend - $this->timestart; } + /** + * error_log + * @param mixed $error + * + * @return void + */ public function error_log($error) { // @codingStandardsIgnoreStart error_log($error); // @codingStandardsIgnoreEnd } + /** + * log_lock_timing + * @param mixed $lock + * + * @return void + */ public function log_lock_timing($lock) { return; } + /** + * log_object_read + * @param string $readname + * @param string $objectpath + * @param int $objectsize + * + * @return void + */ abstract public function log_object_read($readname, $objectpath, $objectsize = 0); + + /** + * log_object_move + * @param mixed $movename + * @param string $initallocation + * @param string $finallocation + * @param string $objecthash + * @param int $objectsize + * + * @return void + */ abstract public function log_object_move($movename, $initallocation, $finallocation, $objecthash, $objectsize = 0); + + /** + * log_object_query + * @param string $queryname + * @param int $objectcount + * @param int $objectsum + * + * @return void + */ abstract public function log_object_query($queryname, $objectcount, $objectsum = 0); } diff --git a/classes/log/objectfs_statistic.php b/classes/log/objectfs_statistic.php index 320cbf66..44c5279a 100644 --- a/classes/log/objectfs_statistic.php +++ b/classes/log/objectfs_statistic.php @@ -29,35 +29,78 @@ require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); +/** + * [Description objectfs_statistic] + */ class objectfs_statistic { + /** + * @var string + */ private $key; + + /** + * @var int + */ private $objectcount; + + /** + * @var int + */ private $objectsum; + /** + * construct + * @param string $key + */ public function __construct($key) { $this->key = $key; $this->objectcount = 0; $this->objectsum = 0; } + /** + * get_objectcount + * @return int + */ public function get_objectcount() { return $this->objectcount; } + /** + * get_objectsum + * @return int + */ public function get_objectsum() { return $this->objectsum; } + /** + * get_key + * @return string + */ public function get_key() { return $this->key; } + /** + * add_statistic + * @param objectfs_statistic $statistic + * + * @return void + */ public function add_statistic(objectfs_statistic $statistic) { $this->objectcount += $statistic->get_objectcount(); $this->objectsum += $statistic->get_objectsum(); } + /** + * add_object_data + * @param int $objectcount + * @param int $objectsum + * + * @return void + */ public function add_object_data($objectcount, $objectsum) { $this->objectcount += $objectcount; $this->objectsum += $objectsum; diff --git a/classes/log/real_time_logger.php b/classes/log/real_time_logger.php index ae9463d1..e7eca947 100644 --- a/classes/log/real_time_logger.php +++ b/classes/log/real_time_logger.php @@ -29,16 +29,41 @@ require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); +/** + * [Description real_time_logger] + */ class real_time_logger extends objectfs_logger { + /** + * log_object_read_action + * @param string $actionname + * @param string $objectpath + * + * @return mixed + */ public function log_object_read_action($actionname, $objectpath) { } + /** + * log_object_move_action + * @param string $actionname + * @param string $objecthash + * @param string $initallocation + * @param string $finallocation + * + * @return mixed + */ public function log_object_move_action($actionname, $objecthash, $initallocation, $finallocation) { } + /** + * append_timing_string + * @param mixed $logstring + * + * @return void + */ protected function append_timing_string(&$logstring) { $timetaken = $this->get_timing(); if ($timetaken > 0) { @@ -47,6 +72,13 @@ protected function append_timing_string(&$logstring) { } + /** + * append_size_string + * @param string $logstring + * @param int $objectsize + * + * @return void + */ protected function append_size_string(&$logstring, $objectsize) { if ($objectsize > 0) { $objectsize = display_size($objectsize); @@ -54,6 +86,14 @@ protected function append_size_string(&$logstring, $objectsize) { } } + /** + * append_location_change_string + * @param string $logstring + * @param string $initiallocation + * @param string $finallocation + * + * @return void + */ protected function append_location_change_string(&$logstring, $initiallocation, $finallocation) { if ($initiallocation == $finallocation) { $logstring .= "The object location did not change from $initiallocation. "; @@ -62,6 +102,14 @@ protected function append_location_change_string(&$logstring, $initiallocation, } } + /** + * log_object_read + * @param string $readname + * @param string $objectpath + * @param int $objectsize + * + * @return [type] + */ public function log_object_read($readname, $objectpath, $objectsize = 0) { $logstring = "The read action '$readname' was used on object with path $objectpath. "; $this->append_timing_string($logstring); @@ -73,6 +121,16 @@ public function log_object_read($readname, $objectpath, $objectsize = 0) { // @codingStandardsIgnoreEnd } + /** + * log_object_move + * @param string $movename + * @param string $initallocation + * @param string $finallocation + * @param string $objecthash + * @param int $objectsize + * + * @return void + */ public function log_object_move($movename, $initallocation, $finallocation, $objecthash, $objectsize = 0) { $logstring = "The move action '$movename' was performed on object with hash $objecthash. "; $this->append_location_change_string($logstring, $initallocation, $finallocation); @@ -83,6 +141,14 @@ public function log_object_move($movename, $initallocation, $finallocation, $obj // @codingStandardsIgnoreEnd } + /** + * log_object_query + * @param string $queryname + * @param int $objectcount + * @param int $objectsum + * + * @return void + */ public function log_object_query($queryname, $objectcount, $objectsum = 0) { $logstring = "The query action '$queryname' was performed. $objectcount objects were returned"; $this->append_timing_string($logstring); @@ -91,6 +157,12 @@ public function log_object_query($queryname, $objectcount, $objectsum = 0) { // @codingStandardsIgnoreEnd } + /** + * log_lock_timing + * @param mixed $lock + * + * @return void + */ public function log_lock_timing($lock) { $locktime = $this->get_timing(); if ($lock) { diff --git a/classes/privacy/provider.php b/classes/privacy/provider.php index fb0cca49..ff971d00 100644 --- a/classes/privacy/provider.php +++ b/classes/privacy/provider.php @@ -26,7 +26,7 @@ use core_privacy\local\legacy_polyfill; /** * Class provider - * @package tool_objectfs\privacy + * @package tool_objectfs */ class provider implements null_provider { use legacy_polyfill; @@ -36,7 +36,7 @@ class provider implements null_provider { * * @return string */ - public static function _get_reason() { + public static function get_reason(): string { return 'privacy:metadata'; } } diff --git a/classes/s3_file_system.php b/classes/s3_file_system.php index 7ebc0dea..727b27a2 100644 --- a/classes/s3_file_system.php +++ b/classes/s3_file_system.php @@ -30,6 +30,9 @@ use tool_objectfs\local\store\s3\file_system; +/** + * [Description s3_file_system] + */ class s3_file_system extends file_system { } diff --git a/classes/swift_file_system.php b/classes/swift_file_system.php index 099ca8a5..a1ef4239 100644 --- a/classes/swift_file_system.php +++ b/classes/swift_file_system.php @@ -27,6 +27,9 @@ use tool_objectfs\local\store\swift\file_system; +/** + * [Description swift_file_system] + */ class swift_file_system extends file_system { } diff --git a/classes/task/check_objects_location.php b/classes/task/check_objects_location.php index 1f86729a..51365829 100644 --- a/classes/task/check_objects_location.php +++ b/classes/task/check_objects_location.php @@ -27,6 +27,9 @@ use tool_objectfs\local\object_manipulator\checker; +/** + * [Description check_objects_location] + */ class check_objects_location extends task { /** @var string $manipulator */ diff --git a/classes/task/delete_local_empty_directories.php b/classes/task/delete_local_empty_directories.php index 70a026a6..3fd8fd05 100644 --- a/classes/task/delete_local_empty_directories.php +++ b/classes/task/delete_local_empty_directories.php @@ -28,8 +28,9 @@ use coding_exception; use tool_objectfs\local\manager; -defined('MOODLE_INTERNAL') || die(); - +/** + * [Description delete_local_empty_directories] + */ class delete_local_empty_directories extends task { /** @var string $stringname */ diff --git a/classes/task/delete_local_objects.php b/classes/task/delete_local_objects.php index c9b9af77..326dc02a 100644 --- a/classes/task/delete_local_objects.php +++ b/classes/task/delete_local_objects.php @@ -28,6 +28,9 @@ use tool_objectfs\local\object_manipulator\deleter; +/** + * [Description delete_local_objects] + */ class delete_local_objects extends task { /** @var string $manipulator */ diff --git a/classes/task/delete_orphaned_object_metadata.php b/classes/task/delete_orphaned_object_metadata.php index c631bafd..e265dcfb 100644 --- a/classes/task/delete_orphaned_object_metadata.php +++ b/classes/task/delete_orphaned_object_metadata.php @@ -15,7 +15,9 @@ // along with Moodle. If not, see . /** - * Task that checks for old orphaned objects, and removes their metadata (record) + * Task that checks for old orphaned objects + * + * And removes their metadata (record) * and external file (if delete external enabled) as it is no longer useful/relevant. * * @package tool_objectfs @@ -30,6 +32,9 @@ require_once(__DIR__ . '/../../lib.php'); +/** + * [Description delete_orphaned_object_metadata] + */ class delete_orphaned_object_metadata extends task { /** @var string $stringname */ diff --git a/classes/task/generate_status_report.php b/classes/task/generate_status_report.php index 2e11e59d..50d1bc4a 100644 --- a/classes/task/generate_status_report.php +++ b/classes/task/generate_status_report.php @@ -31,6 +31,9 @@ require_once(__DIR__ . '/../../lib.php'); +/** + * [Description generate_status_report] + */ class generate_status_report extends task { /** @var string $stringname */ @@ -38,6 +41,7 @@ class generate_status_report extends task { /** * Execute task + * @return void */ public function execute() { objectfs_report::cleanup_reports(); diff --git a/classes/task/orphan_objects.php b/classes/task/orphan_objects.php index 8ca86cbd..23ae7803 100644 --- a/classes/task/orphan_objects.php +++ b/classes/task/orphan_objects.php @@ -15,8 +15,7 @@ // along with Moodle. If not, see . /** - * Task that orphans {tool_objectfs_object} records for deleted - * {files} records. + * Task that orphans {tool_objectfs_object} records for deleted {files} records. * * @package tool_objectfs * @author Nathan Mares @@ -29,6 +28,9 @@ use tool_objectfs\local\object_manipulator\orphaner; +/** + * [Description orphan_objects] + */ class orphan_objects extends task { /** @var string $manipulator */ diff --git a/classes/task/pull_objects_from_storage.php b/classes/task/pull_objects_from_storage.php index 64b89bb3..a6a0875c 100644 --- a/classes/task/pull_objects_from_storage.php +++ b/classes/task/pull_objects_from_storage.php @@ -27,6 +27,9 @@ use tool_objectfs\local\object_manipulator\puller; +/** + * [Description pull_objects_from_storage] + */ class pull_objects_from_storage extends task { /** @var string $manipulator */ diff --git a/classes/task/push_objects_to_storage.php b/classes/task/push_objects_to_storage.php index 0fe5fad3..fbc3c5e8 100644 --- a/classes/task/push_objects_to_storage.php +++ b/classes/task/push_objects_to_storage.php @@ -27,6 +27,9 @@ use tool_objectfs\local\object_manipulator\pusher; +/** + * [Description push_objects_to_storage] + */ class push_objects_to_storage extends task { /** @var string $manipulator */ diff --git a/classes/task/recover_error_objects.php b/classes/task/recover_error_objects.php index 9eae2d14..dd392bc9 100644 --- a/classes/task/recover_error_objects.php +++ b/classes/task/recover_error_objects.php @@ -27,6 +27,9 @@ use tool_objectfs\local\object_manipulator\recoverer; +/** + * [Description recover_error_objects] + */ class recover_error_objects extends task { /** @var string $manipulator */ diff --git a/classes/task/task.php b/classes/task/task.php index 021cc021..40031999 100644 --- a/classes/task/task.php +++ b/classes/task/task.php @@ -35,6 +35,9 @@ require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); +/** + * [Description task] + */ abstract class task extends \core\task\scheduled_task implements objectfs_task { /** @var stdClass $config */ @@ -68,6 +71,7 @@ public function execute() { } /** + * enabled_tasks * @return bool * @throws coding_exception */ diff --git a/classes/tests/test_azure_integration_client.php b/classes/tests/test_azure_integration_client.php index cb77a825..cec69065 100644 --- a/classes/tests/test_azure_integration_client.php +++ b/classes/tests/test_azure_integration_client.php @@ -18,16 +18,34 @@ use tool_objectfs\local\store\azure\client; +/** + * [Description test_azure_integration_client] + * @package tool_objectfs + */ class test_azure_integration_client extends client { + /** + * @var string + */ private $runidentifier; + /** + * construct + * @param mixed $config + * @return void + */ public function __construct($config) { parent::__construct($config); $time = microtime(); $this->runidentifier = md5($time); } + /** + * get_filepath_from_hash + * @param mixed $contenthash + * + * @return string + */ protected function get_filepath_from_hash($contenthash) { $l1 = $contenthash[0] . $contenthash[1]; $l2 = $contenthash[2] . $contenthash[3]; diff --git a/classes/tests/test_client.php b/classes/tests/test_client.php index c7d4ce53..7a380c17 100644 --- a/classes/tests/test_client.php +++ b/classes/tests/test_client.php @@ -18,12 +18,26 @@ use tool_objectfs\local\store\object_client_base; +/** + * [Description test_client] + * @package tool_objectfs + */ class test_client extends object_client_base { - /** @var int $maxupload Maximum allowed file size that can be uploaded. */ + /** + * Maximum allowed file size that can be uploaded + * @var int + */ protected $maxupload; + /** + * @var string + */ private $bucketpath; + /** + * string + * @param \stdClass $config + */ public function __construct($config) { global $CFG; $this->maxupload = 5000000000; @@ -40,33 +54,73 @@ public function __construct($config) { } } + /** + * get_seekable_stream_context + * @return resource + */ public function get_seekable_stream_context() { $context = stream_context_create(); return $context; } + /** + * get_fullpath_from_hash + * @param string $contenthash + * + * @return string + */ public function get_fullpath_from_hash($contenthash) { return "$this->bucketpath/{$contenthash}"; } + /** + * delete_file + * @param string $fullpath + * + * @return bool + */ public function delete_file($fullpath) { return unlink($fullpath); } + /** + * rename_file + * @param string $currentpath + * @param string $destinationpath + * + * @return bool + */ public function rename_file($currentpath, $destinationpath) { return rename($currentpath, $destinationpath); } + /** + * register_stream_wrapper + * @return bool + */ public function register_stream_wrapper() { return true; } + /** + * get_md5_from_hash + * @param mixed $contenthash + * + * @return string + */ private function get_md5_from_hash($contenthash) { $path = $this->get_fullpath_from_hash($contenthash); return md5_file($path); } + /** + * verify_object + * @param string $contenthash + * @param string $localpath + * + * @return bool + */ public function verify_object($contenthash, $localpath) { // For objects uploaded to S3 storage using the multipart upload, the etag will not be the objects MD5. // So we can't compare here to verify the object. @@ -78,14 +132,28 @@ public function verify_object($contenthash, $localpath) { return false; } + /** + * test_connection + * @return \stdClass + */ public function test_connection() { return (object)['success' => true, 'details' => '']; } + /** + * test_permissions + * @param mixed $testdelete + * + * @return \stdClass + */ public function test_permissions($testdelete) { return (object)['success' => true, 'details' => '']; } + /** + * test_permissions + * @return int + */ public function get_maximum_upload_size() { return $this->maxupload; } diff --git a/classes/tests/test_digitalocean_integration_client.php b/classes/tests/test_digitalocean_integration_client.php index 341cdd25..2a34c28c 100644 --- a/classes/tests/test_digitalocean_integration_client.php +++ b/classes/tests/test_digitalocean_integration_client.php @@ -18,16 +18,34 @@ use tool_objectfs\local\store\digitalocean\client; +/** + * [Description test_digitalocean_integration_client] + * @package tool_objectfs + */ class test_digitalocean_integration_client extends client { + /** + * @var string + */ private $runidentifier; + /** + * construct + * @param mixed $config + * @return void + */ public function __construct($config) { parent::__construct($config); $time = microtime(); $this->runidentifier = md5($time); } + /** + * get_filepath_from_hash + * @param mixed $contenthash + * + * @return string + */ protected function get_filepath_from_hash($contenthash) { $l1 = $contenthash[0] . $contenthash[1]; $l2 = $contenthash[2] . $contenthash[3]; diff --git a/classes/tests/test_file_system.php b/classes/tests/test_file_system.php index 21d85d6e..7e1ba875 100644 --- a/classes/tests/test_file_system.php +++ b/classes/tests/test_file_system.php @@ -31,10 +31,22 @@ use tool_objectfs\local\manager; use tool_objectfs\local\store\object_file_system; +/** + * [Description test_file_system] + */ class test_file_system extends object_file_system { + /** + * @var int + */ private $maxupload; + /** + * initialise_external_client + * @param \stdClass $config + * + * @return mixed + */ protected function initialise_external_client($config) { global $CFG; if (isset($CFG->phpunit_objectfs_s3_integration_test_credentials)) { @@ -71,6 +83,7 @@ protected function initialise_external_client($config) { } /** + * get_maximum_upload_size * @return float|int */ public function get_maximum_upload_size() { diff --git a/classes/tests/test_s3_integration_client.php b/classes/tests/test_s3_integration_client.php index 0dc0d0d3..233f0b7e 100644 --- a/classes/tests/test_s3_integration_client.php +++ b/classes/tests/test_s3_integration_client.php @@ -18,16 +18,33 @@ use tool_objectfs\local\store\s3\client; +/** + * [Description test_s3_integration_client] + * @package tool_objectfs + */ class test_s3_integration_client extends client { + /** + * @var string + */ private $runidentifier; + /** + * construct + * @param \stdClass $config + */ public function __construct($config) { parent::__construct($config); $time = microtime(); $this->runidentifier = md5($time); } + /** + * get_filepath_from_hash + * @param string $contenthash + * + * @return string + */ protected function get_filepath_from_hash($contenthash) { $l1 = $contenthash[0] . $contenthash[1]; $l2 = $contenthash[2] . $contenthash[3]; diff --git a/classes/tests/test_swift_integration_client.php b/classes/tests/test_swift_integration_client.php index fd733ca8..4e1ede63 100644 --- a/classes/tests/test_swift_integration_client.php +++ b/classes/tests/test_swift_integration_client.php @@ -18,16 +18,33 @@ use tool_objectfs\local\store\swift\client; +/** + * [Description test_swift_integration_client] + * @package tool_objectfs + */ class test_swift_integration_client extends client { + /** + * @var string + */ private $runidentifier; + /** + * string + * @param \stdClass $config + */ public function __construct($config) { parent::__construct($config); $time = microtime(); $this->runidentifier = md5($time); } + /** + * get_filepath_from_hash + * @param string $contenthash + * + * @return string + */ protected function get_filepath_from_hash($contenthash) { $l1 = $contenthash[0] . $contenthash[1]; $l2 = $contenthash[2] . $contenthash[3]; diff --git a/classes/tests/testcase.php b/classes/tests/testcase.php index 0d553dbb..56e0137d 100644 --- a/classes/tests/testcase.php +++ b/classes/tests/testcase.php @@ -25,6 +25,10 @@ use tool_objectfs\local\store\object_file_system; use tool_objectfs\local\store\signed_url; +/** + * [Description testcase] + * @package tool_objectfs + */ abstract class testcase extends \advanced_testcase { /** @var test_file_system Filesystem */ @@ -33,6 +37,10 @@ abstract class testcase extends \advanced_testcase { /** @var \tool_objectfs\log\objectfs_logger Logger */ public $logger; + /** + * setUp + * @return void + */ protected function setUp(): void { global $CFG; $CFG->alternative_file_system_class = '\\tool_objectfs\\tests\\test_file_system'; @@ -42,10 +50,20 @@ protected function setUp(): void { $this->resetAfterTest(true); } + /** + * reset_file_system + * @return void + */ protected function reset_file_system() { $this->filesystem = new test_file_system(); } + /** + * create_local_file_from_path + * @param string $pathname + * + * @return stored_file + */ protected function create_local_file_from_path($pathname) { $fs = get_file_storage(); $syscontext = \context_system::instance(); @@ -70,6 +88,12 @@ protected function create_local_file_from_path($pathname) { return $file; } + /** + * create_local_file + * @param string $content + * + * @return stored_file + */ protected function create_local_file($content = 'test content') { $fs = get_file_storage(); $syscontext = \context_system::instance(); @@ -94,6 +118,12 @@ protected function create_local_file($content = 'test content') { return $file; } + /** + * create_duplicated_file + * @param string $content + * + * @return stored_file + */ protected function create_duplicated_file($content = 'test content') { $file = $this->create_local_file($content); $contenthash = $file->get_contenthash(); @@ -102,6 +132,12 @@ protected function create_duplicated_file($content = 'test content') { return $file; } + /** + * create_remote_file + * @param string $content + * + * @return stored_file + */ protected function create_remote_file($content = 'test content') { $file = $this->create_duplicated_file($content); $contenthash = $file->get_contenthash(); @@ -110,6 +146,10 @@ protected function create_remote_file($content = 'test content') { return $file; } + /** + * create_error_file + * @return stored_file + */ protected function create_error_file() { $file = $this->create_local_file(); $path = $this->get_local_path_from_storedfile($file); @@ -118,88 +158,188 @@ protected function create_error_file() { return $file; } + /** + * get_external_path_from_hash + * @param string $contenthash + * + * @return mixed + */ protected function get_external_path_from_hash($contenthash) { $reflection = new \ReflectionMethod(object_file_system::class, 'get_external_path_from_hash'); $reflection->setAccessible(true); return $reflection->invokeArgs($this->filesystem, [$contenthash]); } + /** + * get_external_path_from_storedfile + * @param mixed $file + * + * @return mixed + */ protected function get_external_path_from_storedfile($file) { $contenthash = $file->get_contenthash(); return $this->get_external_path_from_hash($contenthash); } - // We want acces to local path for testing so we use a reflection method as opposed to rewriting here. + /** + * get_local_path_from_hash + * + * We want acces to local path for testing so we use a reflection + * method as opposed to rewriting here. + * @param string $contenthash + * + * @return mixed + */ protected function get_local_path_from_hash($contenthash) { $reflection = new \ReflectionMethod(object_file_system::class, 'get_local_path_from_hash'); $reflection->setAccessible(true); return $reflection->invokeArgs($this->filesystem, [$contenthash]); } + /** + * delete_file + * @param string $contenthash + * + * @return mixed + */ protected function delete_file($contenthash) { $reflection = new \ReflectionMethod(object_file_system::class, 'delete_file'); $reflection->setAccessible(true); return $reflection->invokeArgs($this->filesystem, [$contenthash]); } + /** + * rename_file + * @param mixed $currentpath + * @param mixed $destinationpath + * + * @return mixed + */ protected function rename_file($currentpath, $destinationpath) { $reflection = new \ReflectionMethod(object_file_system::class, 'rename_file'); $reflection->setAccessible(true); return $reflection->invokeArgs($this->filesystem, [$currentpath, $destinationpath]); } + /** + * get_local_path_from_storedfile + * @param mixed $file + * + * @return mixed + */ protected function get_local_path_from_storedfile($file) { $contenthash = $file->get_contenthash(); return $this->get_local_path_from_hash($contenthash); } + /** + * recover_file + * @param mixed $file + * + * @return mixed + */ protected function recover_file($file) { $reflection = new \ReflectionMethod(object_file_system::class, 'recover_file'); $reflection->setAccessible(true); return $reflection->invokeArgs($this->filesystem, [$file]); } + /** + * create_local_object + * @param mixed $content + * + * @return mixed + */ protected function create_local_object($content = 'local object content') { $file = $this->create_local_file($content); return $this->create_object_record($file, OBJECT_LOCATION_LOCAL); } + /** + * create_duplicated_object + * @param mixed $content + * + * @return mixed + */ protected function create_duplicated_object($content = 'duplicated object content') { $file = $this->create_duplicated_file($content); return $this->create_object_record($file, OBJECT_LOCATION_DUPLICATED); } + /** + * create_remote_object + * @param mixed $content + * + * @return mixed + */ protected function create_remote_object($content = 'remote object content') { $file = $this->create_remote_file($content); return $this->create_object_record($file, OBJECT_LOCATION_EXTERNAL); } + /** + * create_error_object + * @param mixed $content + * + * @return mixed + */ protected function create_error_object($content = 'error object content') { $file = $this->create_error_file($content); return $this->create_object_record($file, OBJECT_LOCATION_ERROR); } + /** + * is_locally_readable_by_hash + * @param mixed $contenthash + * + * @return mixed + */ protected function is_locally_readable_by_hash($contenthash) { $localpath = $this->get_local_path_from_hash($contenthash); return is_readable($localpath); } + /** + * is_externally_readable_by_hash + * @param mixed $contenthash + * + * @return mixed + */ protected function is_externally_readable_by_hash($contenthash) { $externalpath = $this->get_external_path_from_hash($contenthash); return is_readable($externalpath); } + /** + * acquire_object_lock + * @param mixed $filehash + * @param int $timeout + * + * @return mixed + */ protected function acquire_object_lock($filehash, $timeout = 0) { $reflection = new \ReflectionMethod(object_file_system::class, 'acquire_object_lock'); $reflection->setAccessible(true); return $reflection->invokeArgs($this->filesystem, [$filehash, $timeout]); } + /** + * delete_draft_files + * @param mixed $contenthash + * + * @return mixed + */ protected function delete_draft_files($contenthash) { global $DB; $DB->delete_records('files', array('contenthash' => $contenthash)); } + /** + * is_externally_readable_by_url + * @param signed_url $signedurl + * + * @return mixed + */ protected function is_externally_readable_by_url(signed_url $signedurl) { try { $file = fopen($signedurl->url->out(false), 'r'); @@ -216,8 +356,9 @@ protected function is_externally_readable_by_url(signed_url $signedurl) { } /** + * create_object_record * @param stored_file $file - * @param $location + * @param mixed $location * @return stdClass * @throws dml_exception */ @@ -233,6 +374,7 @@ private function create_object_record(stored_file $file, $location) { } /** + * objects_contain_hash * @param string $contenthash * @return bool * @throws moodle_exception diff --git a/db/upgrade.php b/db/upgrade.php index c54d30c9..d9f697fe 100644 --- a/db/upgrade.php +++ b/db/upgrade.php @@ -23,6 +23,12 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ +/** + * xmldb_tool_objectfs_upgrade + * @param int $oldversion + * + * @return bool + */ function xmldb_tool_objectfs_upgrade($oldversion) { global $DB; $dbman = $DB->get_manager(); diff --git a/lib.php b/lib.php index 5106bcc3..6402c1fa 100644 --- a/lib.php +++ b/lib.php @@ -71,8 +71,8 @@ /** * Sends a plugin file to the browser. - * @param $course - * @param $cm + * @param mixed $course + * @param mixed $cm * @param \context $context * @param string $filearea * @param array $args diff --git a/presignedurl_tests.php b/presignedurl_tests.php index 1ac1bc66..cc40e3e4 100644 --- a/presignedurl_tests.php +++ b/presignedurl_tests.php @@ -58,7 +58,7 @@ $deletelinktext = get_string('settings:presignedurl:deletefiles', OBJECTFS_PLUGIN_NAME); echo $output->heading(html_writer::link($deleteurl, $deletelinktext) . $deletedsuccess, 6); $client = manager::get_client($config); - if ($client and $client->get_availability()) { + if ($client && $client->get_availability()) { $connection = $client->test_connection(); if ($connection->success) { $testfiles = $output->presignedurl_tests_load_files($fs); diff --git a/renderer.php b/renderer.php index 7f177715..5fa571fb 100644 --- a/renderer.php +++ b/renderer.php @@ -27,6 +27,9 @@ use tool_objectfs\local\report\objectfs_report; use tool_objectfs\local\store\object_file_system; +/** + * [Description tool_objectfs_renderer] + */ class tool_objectfs_renderer extends plugin_renderer_base { /** @@ -50,6 +53,12 @@ public function delete_presignedurl_tests_files() { } } + /** + * presignedurl_tests_load_files + * @param mixed $fs + * + * @return array + */ public function presignedurl_tests_load_files($fs) { global $CFG; $filestorage = get_file_storage(); @@ -95,6 +104,13 @@ public function presignedurl_tests_load_files($fs) { return $testfiles; } + /** + * presignedurl_tests_content + * @param mixed $fs + * @param mixed $testfiles + * + * @return string + */ public function presignedurl_tests_content($fs, $testfiles) { global $CFG; $CFG->enablepresignedurls = true; diff --git a/tests/local/object_manipulator/checker_test.php b/tests/local/object_manipulator/checker_test.php index 32385996..5f24de71 100644 --- a/tests/local/object_manipulator/checker_test.php +++ b/tests/local/object_manipulator/checker_test.php @@ -22,6 +22,7 @@ * Tests for object checker. * * @covers \tool_objectfs\local\object_manipulator\checker + * @package tool_objectfs */ class checker_test extends \tool_objectfs\tests\testcase { diff --git a/tests/local/object_manipulator/deleter_test.php b/tests/local/object_manipulator/deleter_test.php index 2588d999..a29499f8 100644 --- a/tests/local/object_manipulator/deleter_test.php +++ b/tests/local/object_manipulator/deleter_test.php @@ -22,6 +22,7 @@ * Tests for object deleter. * * @covers \tool_objectfs\local\object_manipulator\deleter + * @package tool_objectfs */ class deleter_test extends \tool_objectfs\tests\testcase { @@ -47,6 +48,13 @@ protected function tearDown(): void { ob_end_clean(); } + /** + * set_deleter_config + * @param mixed $key + * @param mixed $value + * + * @return void + */ protected function set_deleter_config($key, $value) { $config = manager::get_objectfs_config(); $config->$key = $value; diff --git a/tests/local/object_manipulator/orphaner_test.php b/tests/local/object_manipulator/orphaner_test.php index e5f7a04b..6befdf30 100644 --- a/tests/local/object_manipulator/orphaner_test.php +++ b/tests/local/object_manipulator/orphaner_test.php @@ -23,6 +23,7 @@ * Tests for object orphaner. * * @covers \tool_objectfs\local\object_manipulator\orphaner + * @package tool_objectfs */ class orphaner_test extends \tool_objectfs\tests\testcase { @@ -47,6 +48,13 @@ protected function tearDown(): void { ob_end_clean(); } + /** + * set_orphaner_config + * @param mixed $key + * @param mixed $value + * + * @return void + */ protected function set_orphaner_config($key, $value) { $config = manager::get_objectfs_config(); $config->$key = $value; diff --git a/tests/local/object_manipulator/puller_test.php b/tests/local/object_manipulator/puller_test.php index 04b6e5da..0c793613 100644 --- a/tests/local/object_manipulator/puller_test.php +++ b/tests/local/object_manipulator/puller_test.php @@ -22,6 +22,7 @@ * Tests for object puller. * * @covers \tool_objectfs\local\object_manipulator\puller + * @package tool_objectfs */ class puller_test extends \tool_objectfs\tests\testcase { @@ -45,6 +46,13 @@ protected function tearDown(): void { ob_end_clean(); } + /** + * set_puller_config + * @param mixed $key + * @param mixed $value + * + * @return void + */ protected function set_puller_config($key, $value) { $config = manager::get_objectfs_config(); $config->$key = $value; diff --git a/tests/local/object_manipulator/pusher_test.php b/tests/local/object_manipulator/pusher_test.php index e7c1f3db..3c1c93a1 100644 --- a/tests/local/object_manipulator/pusher_test.php +++ b/tests/local/object_manipulator/pusher_test.php @@ -23,6 +23,7 @@ * Tests for object pusher. * * @covers \tool_objectfs\local\object_manipulator\pusher + * @package tool_objectfs */ class pusher_test extends \tool_objectfs\tests\testcase { @@ -47,6 +48,13 @@ protected function tearDown(): void { ob_end_clean(); } + /** + * set_pusher_config + * @param mixed $key + * @param mixed $value + * + * @return void + */ protected function set_pusher_config($key, $value) { $config = manager::get_objectfs_config(); $config->$key = $value; diff --git a/tests/local/object_manipulator/recoverer_test.php b/tests/local/object_manipulator/recoverer_test.php index 8a4ec5ba..9a823555 100644 --- a/tests/local/object_manipulator/recoverer_test.php +++ b/tests/local/object_manipulator/recoverer_test.php @@ -23,6 +23,7 @@ * Tests for object recoverer. * * @covers \tool_objectfs\local\object_manipulator\recoverer + * @package tool_objectfs */ class recoverer_test extends \tool_objectfs\tests\testcase { diff --git a/tests/local/tasks_test.php b/tests/local/tasks_test.php index 8a71d024..afdc53b2 100644 --- a/tests/local/tasks_test.php +++ b/tests/local/tasks_test.php @@ -20,6 +20,7 @@ * End to end tests for tasks. Make sure all the plumbing is ok. * * @covers \tool_objectfs\local\manager + * @package tool_objectfs */ class tasks_test extends \tool_objectfs\tests\testcase { diff --git a/tests/object_file_system_test.php b/tests/object_file_system_test.php index af3c4105..428db219 100644 --- a/tests/object_file_system_test.php +++ b/tests/object_file_system_test.php @@ -24,9 +24,17 @@ * Test basic operations of object file system. * * @covers \tool_objectfs\local\store\object_file_system + * @package tool_objectfs */ class object_file_system_test extends tests\testcase { + /** + * set_externalclient_config + * @param mixed $key + * @param mixed $value + * + * @return void + */ public function set_externalclient_config($key, $value) { // Get a reflection of externalclient object as a property. $reflection = new \ReflectionClass($this->filesystem); @@ -223,6 +231,7 @@ public function test_delete_object_from_local_by_hash_if_can_verify_external_obj } /** + * delete_empty_folders_provider * @return array */ public function delete_empty_folders_provider() { @@ -268,6 +277,7 @@ public function delete_empty_folders_provider() { } /** + * test_delete_empty_folders_provider * @dataProvider delete_empty_folders_provider * @param array $dirs Dirs to be created. * @param array $files Files to be created. @@ -364,6 +374,10 @@ public function test_get_content_updates_object_with_error_location_on_fail() { $this->assertEquals(OBJECT_LOCATION_ERROR, $location); } + /** + * error_surpressor + * @return void + */ public function error_surpressor() { // We do nothing. We cant surpess warnings // normally because phpunit will still fail. @@ -636,6 +650,10 @@ public function test_presigned_url_configured_method_returns_true_if_configured( $this->assertTrue($this->filesystem->presigned_url_configured()); } + /** + * presigned_url_should_redirect_provider + * @return array + */ public function presigned_url_should_redirect_provider() { $provider = array(); @@ -675,11 +693,12 @@ public function presigned_url_should_redirect_provider() { } /** - * @dataProvider presigned_url_should_redirect_provider + * test_presigned_url_should_redirect_provider * - * @param $enablepresignedurls mixed enable pre-signed URLs. - * @param $presignedminfilesize mixed minimum file size to be redirected to pre-signed URL. - * @param $result boolean expected result. + * @dataProvider presigned_url_should_redirect_provider + * @param mixed $enablepresignedurls enable pre-signed URLs. + * @param mixed $presignedminfilesize minimum file size to be redirected to pre-signed URL. + * @param bool $result expected result. * @throws \dml_exception */ public function test_presigned_url_should_redirect_method_with_data_provider($enablepresignedurls, @@ -737,7 +756,8 @@ public function get_expiration_time_method_if_supported_provider() { [7200, $now, userdate($now - 100, '%a, %d %b %Y %H:%M:%S'), $now + (2 * MINSECS) - $secondsafternowsub100], [7200, $now, userdate($now + 30, '%a, %d %b %Y %H:%M:%S'), $now + (2 * MINSECS) - $secondsafternowadd30], [7200, $now, userdate($now + 100, '%a, %d %b %Y %H:%M:%S'), $now + (2 * MINSECS) - $secondsafternowadd100], - [7200, $now, userdate($now + WEEKSECS + HOURSECS, '%a, %d %b %Y %H:%M:%S'), $now + WEEKSECS - MINSECS - $secondsafternowaddweek], + [7200, $now, userdate($now + WEEKSECS + HOURSECS, '%a, %d %b %Y %H:%M:%S'), + $now + WEEKSECS - MINSECS - $secondsafternowaddweek], // Custom Pre-Signed URL expiration time and int-like 'Expires' header. [0, $now, 0, $now + (2 * MINSECS) - $secondsafternow], @@ -753,7 +773,8 @@ public function get_expiration_time_method_if_supported_provider() { [600, $now, userdate($now - 100, '%a, %d %b %Y %H:%M:%S'), $now + (2 * MINSECS) - $secondsafternowsub100], [600, $now, userdate($now + 30, '%a, %d %b %Y %H:%M:%S'), $now + (2 * MINSECS) - $secondsafternowadd30], [600, $now, userdate($now + 100, '%a, %d %b %Y %H:%M:%S'), $now + (2 * MINSECS) - $secondsafternowadd100], - [600, $now, userdate($now + WEEKSECS + HOURSECS, '%a, %d %b %Y %H:%M:%S'), $now + WEEKSECS - MINSECS - $secondsafternowaddweek], + [600, $now, userdate($now + WEEKSECS + HOURSECS, '%a, %d %b %Y %H:%M:%S'), + $now + WEEKSECS - MINSECS - $secondsafternowaddweek], ]; } From bf01ea2fc07fa377e3d60401e00556b0be6188ff Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Mon, 29 Jul 2024 15:22:08 +1000 Subject: [PATCH 06/28] cleanup: cleanup to match coding standards --- .../candidates/deleter_candidates.php | 2 +- .../candidates/orphaner_candidates.php | 2 +- .../manipulator_builder.php | 2 +- .../local/report/location_report_builder.php | 6 +- .../report/object_location_history_table.php | 4 +- .../report/object_status_history_table.php | 4 +- classes/local/report/objectfs_report.php | 4 +- classes/local/store/azure/client.php | 12 ++-- classes/local/store/digitalocean/client.php | 12 ++-- classes/local/store/object_client.php | 2 +- classes/local/store/object_client_base.php | 2 +- classes/local/store/object_file_system.php | 6 +- classes/local/store/s3/client.php | 60 ++++++++-------- classes/local/store/swift/client.php | 6 +- classes/log/aggregate_logger.php | 18 ++--- .../task/delete_orphaned_object_metadata.php | 2 +- classes/tests/testcase.php | 10 +-- db/tasks.php | 58 ++++++++-------- db/upgrade.php | 6 +- lib.php | 2 +- object_location.php | 2 +- renderer.php | 22 +++--- tests/local/manager_test.php | 6 +- .../local/object_manipulator/checker_test.php | 16 ++--- .../local/object_manipulator/deleter_test.php | 20 +++--- .../object_manipulator/orphaner_test.php | 2 +- .../local/object_manipulator/puller_test.php | 14 ++-- .../local/object_manipulator/pusher_test.php | 22 +++--- .../object_manipulator/recoverer_test.php | 24 +++---- tests/local/report/object_status_test.php | 4 +- tests/local/store/clients_test.php | 8 +-- tests/object_file_system_test.php | 68 +++++++++---------- tests/task/populate_objects_filesize_test.php | 4 +- version.php | 2 +- 34 files changed, 217 insertions(+), 217 deletions(-) diff --git a/classes/local/object_manipulator/candidates/deleter_candidates.php b/classes/local/object_manipulator/candidates/deleter_candidates.php index 30f7d2df..d85a56bf 100644 --- a/classes/local/object_manipulator/candidates/deleter_candidates.php +++ b/classes/local/object_manipulator/candidates/deleter_candidates.php @@ -57,7 +57,7 @@ public function get_candidates_sql_params() { return [ 'consistancythreshold' => $consistancythreshold, 'location' => OBJECT_LOCATION_DUPLICATED, - 'sizethreshold' => $this->config->sizethreshold + 'sizethreshold' => $this->config->sizethreshold, ]; } } diff --git a/classes/local/object_manipulator/candidates/orphaner_candidates.php b/classes/local/object_manipulator/candidates/orphaner_candidates.php index e50f4806..2703b2af 100644 --- a/classes/local/object_manipulator/candidates/orphaner_candidates.php +++ b/classes/local/object_manipulator/candidates/orphaner_candidates.php @@ -53,7 +53,7 @@ public function get_candidates_sql() { */ public function get_candidates_sql_params() { return [ - 'location' => OBJECT_LOCATION_ORPHANED + 'location' => OBJECT_LOCATION_ORPHANED, ]; } } diff --git a/classes/local/object_manipulator/manipulator_builder.php b/classes/local/object_manipulator/manipulator_builder.php index ca7022c8..cbbce3dc 100644 --- a/classes/local/object_manipulator/manipulator_builder.php +++ b/classes/local/object_manipulator/manipulator_builder.php @@ -48,7 +48,7 @@ class manipulator_builder { pusher::class, recoverer::class, checker::class, - orphaner::class + orphaner::class, ]; /** @var string $manipulatorclass */ diff --git a/classes/local/report/location_report_builder.php b/classes/local/report/location_report_builder.php index fe700198..d44b9cdc 100644 --- a/classes/local/report/location_report_builder.php +++ b/classes/local/report/location_report_builder.php @@ -47,7 +47,7 @@ public function build_report($reportid) { OBJECT_LOCATION_DUPLICATED, OBJECT_LOCATION_EXTERNAL, OBJECT_LOCATION_ORPHANED, - OBJECT_LOCATION_ERROR + OBJECT_LOCATION_ERROR, ]; $totalcount = 0; @@ -90,7 +90,7 @@ public function build_report($reportid) { if ($location !== OBJECT_LOCATION_ORPHANED) { // Process the query normally. - $result = $DB->get_record_sql($sql, array($location)); + $result = $DB->get_record_sql($sql, [$location]); } else if ($location === OBJECT_LOCATION_ORPHANED) { // Start the query from objectfs, for ORPHANED objects, they are not located in the files table. $sql = @@ -101,7 +101,7 @@ public function build_report($reportid) { WHERE o.location = ?) SELECT COALESCE(COUNT(co.contenthash),0) AS objectcount FROM cte_objects co'; - $result = $DB->get_record_sql($sql, array($location)); + $result = $DB->get_record_sql($sql, [$location]); $result->objectsum = 0; } diff --git a/classes/local/report/object_location_history_table.php b/classes/local/report/object_location_history_table.php index 2a2a96a3..be421609 100644 --- a/classes/local/report/object_location_history_table.php +++ b/classes/local/report/object_location_history_table.php @@ -82,7 +82,7 @@ public function __construct() { public function query_db($pagesize, $useinitialsbar = true) { global $DB; $fields = 'CONCAT(reportid, datakey) AS uid, datakey AS location, objectcount AS count, objectsum AS size'; - $conditions = array('reporttype' => 'location'); + $conditions = ['reporttype' => 'location']; $rawrecords = $DB->get_records('tool_objectfs_report_data', $conditions, 'reportid', $fields); $reports = objectfs_report::get_report_ids(); @@ -90,7 +90,7 @@ public function query_db($pagesize, $useinitialsbar = true) { // NOTE: This avoids the need to null coalesce on a non-existing count/size. $emptyrecord = (object)[ 'count' => 0, - 'size' => 0 + 'size' => 0, ]; foreach ($reports as $id => $timecreated) { // Initialises the records to be used, and fallback to an empty one if not found. diff --git a/classes/local/report/object_status_history_table.php b/classes/local/report/object_status_history_table.php index 31191602..906689ce 100644 --- a/classes/local/report/object_status_history_table.php +++ b/classes/local/report/object_status_history_table.php @@ -102,7 +102,7 @@ public function query_db($pagesize, $useinitialsbar = true) { default: $sort = 'heading ASC'; } - $params = array('reporttype' => $this->reporttype, 'reportid' => $this->reportid); + $params = ['reporttype' => $this->reporttype, 'reportid' => $this->reportid]; $fields = 'datakey AS heading, objectcount AS count, objectsum AS size'; $rows = $DB->get_records('tool_objectfs_report_data', $params, $sort, $fields); $this->rawdata = $rows; @@ -196,7 +196,7 @@ public function add_barchart($value, $max, $type, $precision = 0) { if ($max > 0) { $share = round(100 * $value / $max, $precision); } - $htmlparams = array('class' => 'ofs-bar', 'style' => 'width:'.$share.'%'); + $htmlparams = ['class' => 'ofs-bar', 'style' => 'width:'.$share.'%']; switch ($type) { case 'count': diff --git a/classes/local/report/objectfs_report.php b/classes/local/report/objectfs_report.php index 86cd5e8b..cc9eb910 100644 --- a/classes/local/report/objectfs_report.php +++ b/classes/local/report/objectfs_report.php @@ -150,7 +150,7 @@ public static function generate_status_report() { public static function cleanup_reports() { global $DB; $reportdate = time() - YEARSECS; - $params = array('reportdate' => $reportdate); + $params = ['reportdate' => $reportdate]; $reports = $DB->get_records_select('tool_objectfs_reports', 'reportdate < :reportdate', $params, 'id', 'id'); $reportids = array_keys($reports); $DB->delete_records_list('tool_objectfs_reports', 'id', $reportids); @@ -177,7 +177,7 @@ public static function get_report_types() { */ public static function get_report_ids() { global $DB; - $reports = array(); + $reports = []; $records = $DB->get_records('tool_objectfs_reports', null, 'id DESC', 'id, reportdate'); foreach ($records as $record) { $reports[$record->id] = $record->reportdate; diff --git a/classes/local/store/azure/client.php b/classes/local/store/azure/client.php index 2a6e3fa4..c96bdde8 100644 --- a/classes/local/store/azure/client.php +++ b/classes/local/store/azure/client.php @@ -149,11 +149,11 @@ public function get_relative_path_from_fullpath($fullpath) { * @return resource */ public function get_seekable_stream_context() { - $context = stream_context_create(array( - 'blob' => array( - 'seekable' => true - ) - )); + $context = stream_context_create([ + 'blob' => [ + 'seekable' => true, + ], + ]); return $context; } @@ -259,7 +259,7 @@ public function test_connection() { public function test_permissions($testdelete) { $permissions = new \stdClass(); $permissions->success = true; - $permissions->messages = array(); + $permissions->messages = []; try { $result = $this->client->createBlockBlob($this->container, 'permissions_check_file', 'permissions_check_file'); diff --git a/classes/local/store/digitalocean/client.php b/classes/local/store/digitalocean/client.php index 644043fe..226636d6 100644 --- a/classes/local/store/digitalocean/client.php +++ b/classes/local/store/digitalocean/client.php @@ -77,12 +77,12 @@ public function set_client($config) { return; } - $this->client = \Aws\S3\S3Client::factory(array( - 'credentials' => array('key' => $config->do_key, 'secret' => $config->do_secret), + $this->client = \Aws\S3\S3Client::factory([ + 'credentials' => ['key' => $config->do_key, 'secret' => $config->do_secret], 'region' => $config->do_region, 'endpoint' => 'https://' . $config->do_region . '.digitaloceanspaces.com', - 'version' => AWS_API_VERSION - )); + 'version' => AWS_API_VERSION, + ]); } /** @@ -93,13 +93,13 @@ public function set_client($config) { */ public function define_client_section($settings, $config) { - $regionoptions = array( + $regionoptions = [ 'sfo2' => 'sfo2 (San Fransisco)', 'nyc3' => 'nyc3 (New York City)', 'ams3' => 'ams3 (Amsterdam)', 'sgp1' => 'spg1 (Singapore)', 'fra1' => 'fra1 (Frankfurt)', - ); + ]; $settings->add(new \admin_setting_heading('tool_objectfs/do', new \lang_string('settings:do:header', 'tool_objectfs'), '')); diff --git a/classes/local/store/object_client.php b/classes/local/store/object_client.php index 3c32f1ea..9121a824 100644 --- a/classes/local/store/object_client.php +++ b/classes/local/store/object_client.php @@ -98,7 +98,7 @@ public function verify_object($contenthash, $localpath); * * @return mixed */ - public function generate_presigned_url($contenthash, $headers = array()); + public function generate_presigned_url($contenthash, $headers = []); /** * support_presigned_urls diff --git a/classes/local/store/object_client_base.php b/classes/local/store/object_client_base.php index 358a0866..4f03418d 100644 --- a/classes/local/store/object_client_base.php +++ b/classes/local/store/object_client_base.php @@ -103,7 +103,7 @@ public function support_presigned_urls() { * * @throws \coding_exception */ - public function generate_presigned_url($contenthash, $headers = array()) { + public function generate_presigned_url($contenthash, $headers = []) { throw new \coding_exception("Pre-signed URLs not supported"); } diff --git a/classes/local/store/object_file_system.php b/classes/local/store/object_file_system.php index f66b4c4a..fb3da10d 100644 --- a/classes/local/store/object_file_system.php +++ b/classes/local/store/object_file_system.php @@ -470,7 +470,7 @@ public function delete_empty_dirs($rootpath = '') { $pathinfo['filename'], $pathinfo['basename'], $pathinfo['filename'], - $pathinfo['basename'] + $pathinfo['basename'], ]); if (!$exists) { @@ -844,7 +844,7 @@ protected function copy_from_external_to_local($contenthash) { * @return bool * @throws \dml_exception */ - public function redirect_to_presigned_url($contenthash, $headers = array()) { + public function redirect_to_presigned_url($contenthash, $headers = []) { global $FULLME; try { $signedurl = $this->externalclient->generate_presigned_url($contenthash, $headers); @@ -926,7 +926,7 @@ public function presigned_url_should_redirect_file($file) { * @return bool * @throws \dml_exception */ - public function presigned_url_should_redirect($contenthash, $headers = array()) { + public function presigned_url_should_redirect($contenthash, $headers = []) { // Redirect regardless. if ($this->externalclient->presignedminfilesize == 0 && manager::all_extensions_whitelisted()) { diff --git a/classes/local/store/s3/client.php b/classes/local/store/s3/client.php index f029acf7..a3cce110 100644 --- a/classes/local/store/s3/client.php +++ b/classes/local/store/s3/client.php @@ -94,7 +94,7 @@ public function __construct($config) { * @return array */ public function __sleep() { - return array('bucket'); + return ['bucket']; } /** @@ -153,17 +153,17 @@ public function set_client($config) { return; } - $options = array( + $options = [ 'region' => $config->s3_region, - 'version' => AWS_API_VERSION - ); + 'version' => AWS_API_VERSION, + ]; if (empty($config->s3_usesdkcreds)) { - $options['credentials'] = array('key' => $config->s3_key, 'secret' => $config->s3_secret); + $options['credentials'] = ['key' => $config->s3_key, 'secret' => $config->s3_secret]; } if ($config->useproxy) { - $options['http'] = array('proxy' => $this->get_proxy_string()); + $options['http'] = ['proxy' => $this->get_proxy_string()]; } // Support base_url config for aws api compatible endpoints. @@ -199,9 +199,9 @@ private function get_md5_from_hash($contenthash) { try { $key = $this->get_filepath_from_hash($contenthash); - $result = $this->client->headObject(array( + $result = $this->client->headObject([ 'Bucket' => $this->bucket, - 'Key' => $this->bucketkeyprefix . $key)); + 'Key' => $this->bucketkeyprefix . $key]); } catch (\Aws\S3\Exception\S3Exception $e) { return false; } @@ -268,11 +268,11 @@ public function rename_file($currentpath, $destinationpath) { * @return mixed */ public function get_seekable_stream_context() { - $context = stream_context_create(array( - 's3' => array( - 'seekable' => true - ) - )); + $context = stream_context_create([ + 's3' => [ + 'seekable' => true, + ], + ]); return $context; } @@ -306,7 +306,7 @@ public function test_connection() { $connection->success = false; $connection->details = get_string('settings:notconfigured', 'tool_objectfs'); } else { - $this->client->headBucket(array('Bucket' => $this->bucket)); + $this->client->headBucket(['Bucket' => $this->bucket]); } } catch (\Aws\S3\Exception\S3Exception $e) { $connection->success = false; @@ -334,19 +334,19 @@ public function test_connection() { public function test_permissions($testdelete) { $permissions = new \stdClass(); $permissions->success = true; - $permissions->messages = array(); + $permissions->messages = []; if ($this->is_functional()) { $permissions->success = false; - $permissions->messages = array(); + $permissions->messages = []; return $permissions; } try { - $result = $this->client->putObject(array( + $result = $this->client->putObject([ 'Bucket' => $this->bucket, 'Key' => $this->bucketkeyprefix . 'permissions_check_file', - 'Body' => 'test content')); + 'Body' => 'test content']); } catch (\Aws\S3\Exception\S3Exception $e) { $details = $this->get_exception_details($e); $permissions->messages[get_string('settings:writefailure', 'tool_objectfs') . $details] = 'notifyproblem'; @@ -354,9 +354,9 @@ public function test_permissions($testdelete) { } try { - $result = $this->client->getObject(array( + $result = $this->client->getObject([ 'Bucket' => $this->bucket, - 'Key' => $this->bucketkeyprefix . 'permissions_check_file')); + 'Key' => $this->bucketkeyprefix . 'permissions_check_file']); } catch (\Aws\S3\Exception\S3Exception $e) { $errorcode = $e->getAwsErrorCode(); // Write could have failed. @@ -371,7 +371,7 @@ public function test_permissions($testdelete) { try { $result = $this->client->deleteObject([ 'Bucket' => $this->bucket, - 'Key' => $this->bucketkeyprefix . 'permissions_check_file' + 'Key' => $this->bucketkeyprefix . 'permissions_check_file', ]); $permissions->messages[get_string('settings:deletesuccess', 'tool_objectfs')] = 'warning'; $permissions->success = false; @@ -539,7 +539,7 @@ public function support_presigned_urls() { * @return signed_url * @throws \Exception */ - public function generate_presigned_url($contenthash, $headers = array()) { + public function generate_presigned_url($contenthash, $headers = []) { if ('cf' === $this->signingmethod) { return $this->generate_presigned_url_cloudfront($contenthash, $headers); } @@ -828,18 +828,18 @@ public function curl_range_request_to_presigned_url($contenthash, $ranges, $head } catch (\Exception $e) { throw new \coding_exception('Failed to generate pre-signed url: ' . $e->getMessage()); } - $headers = array( + $headers = [ 'Range: bytes=' . $ranges->rangefrom . '-' . $ranges->rangeto, - ); + ]; $curl = new \curl(); - $curl->setopt(array('CURLOPT_HTTP_VERSION' => CURL_HTTP_VERSION_1_1)); - $curl->setopt(array('CURLOPT_RETURNTRANSFER' => true)); - $curl->setopt(array('CURLOPT_SSL_VERIFYPEER' => false)); - $curl->setopt(array('CURLOPT_CONNECTTIMEOUT' => 15)); - $curl->setopt(array('CURLOPT_TIMEOUT' => 15)); + $curl->setopt(['CURLOPT_HTTP_VERSION' => CURL_HTTP_VERSION_1_1]); + $curl->setopt(['CURLOPT_RETURNTRANSFER' => true]); + $curl->setopt(['CURLOPT_SSL_VERIFYPEER' => false]); + $curl->setopt(['CURLOPT_CONNECTTIMEOUT' => 15]); + $curl->setopt(['CURLOPT_TIMEOUT' => 15]); $curl->setHeader($headers); $content = $curl->get($url); - return array('responseheaders' => $curl->getResponse(), 'content' => $content, 'url' => $url); + return ['responseheaders' => $curl->getResponse(), 'content' => $content, 'url' => $url]; } /** diff --git a/classes/local/store/swift/client.php b/classes/local/store/swift/client.php index 1d520fba..bca88228 100644 --- a/classes/local/store/swift/client.php +++ b/classes/local/store/swift/client.php @@ -70,7 +70,7 @@ private function get_endpoint() { 'password' => $this->config->openstack_password, 'domain' => ['id' => 'default'], ], - 'scope' => ['project' => ['id' => $this->config->openstack_projectid]] + 'scope' => ['project' => ['id' => $this->config->openstack_projectid]], ]; if (!isset($this->config->openstack_authtoken['expires_at']) @@ -179,7 +179,7 @@ public function get_seekable_stream_context() { 'endpoint' => $this->config->openstack_authurl, 'region' => $this->config->openstack_region, 'cachedtoken' => $this->config->openstack_authtoken, - ] + ], ]); return $context; } @@ -288,7 +288,7 @@ public function test_connection() { public function test_permissions($testdelete) { $permissions = new \stdClass(); $permissions->success = true; - $permissions->messages = array(); + $permissions->messages = []; $container = $this->get_container(); diff --git a/classes/log/aggregate_logger.php b/classes/log/aggregate_logger.php index c26029a5..c60fdc1d 100644 --- a/classes/log/aggregate_logger.php +++ b/classes/log/aggregate_logger.php @@ -25,7 +25,7 @@ namespace tool_objectfs\log; -use \tool_objectfs\log\objectfs_statistic; +use tool_objectfs\log\objectfs_statistic; defined('MOODLE_INTERNAL') || die(); @@ -64,14 +64,14 @@ class aggregate_logger extends objectfs_logger { */ public function __construct() { parent::__construct(); - $this->movestatistics = array( - OBJECT_LOCATION_ERROR => array(), - OBJECT_LOCATION_LOCAL => array(), - OBJECT_LOCATION_DUPLICATED => array(), - OBJECT_LOCATION_EXTERNAL => array() - ); - $this->readstatistics = array(); - $this->querystatistics = array(); + $this->movestatistics = [ + OBJECT_LOCATION_ERROR => [], + OBJECT_LOCATION_LOCAL => [], + OBJECT_LOCATION_DUPLICATED => [], + OBJECT_LOCATION_EXTERNAL => [], + ]; + $this->readstatistics = []; + $this->querystatistics = []; } /** diff --git a/classes/task/delete_orphaned_object_metadata.php b/classes/task/delete_orphaned_object_metadata.php index e265dcfb..072ede6d 100644 --- a/classes/task/delete_orphaned_object_metadata.php +++ b/classes/task/delete_orphaned_object_metadata.php @@ -54,7 +54,7 @@ public function execute() { $params = [ 'location' => OBJECT_LOCATION_ORPHANED, - 'ageforremoval' => time() - $ageforremoval + 'ageforremoval' => time() - $ageforremoval, ]; if (!empty($this->config->deleteexternal) && $this->config->deleteexternal == TOOL_OBJECTFS_DELETE_EXTERNAL_TRASH) { diff --git a/classes/tests/testcase.php b/classes/tests/testcase.php index 56e0137d..6d85a3b5 100644 --- a/classes/tests/testcase.php +++ b/classes/tests/testcase.php @@ -72,7 +72,7 @@ protected function create_local_file_from_path($pathname) { $itemid = 0; $filepath = '/'; $sourcefield = 'Copyright stuff'; - $filerecord = array( + $filerecord = [ 'contextid' => $syscontext->id, 'component' => $component, 'filearea' => $filearea, @@ -81,7 +81,7 @@ protected function create_local_file_from_path($pathname) { 'filename' => $pathname, 'source' => $sourcefield, 'mimetype' => 'text', - ); + ]; $file = $fs->create_file_from_pathname($filerecord, $pathname); manager::update_object_by_hash($file->get_contenthash(), OBJECT_LOCATION_LOCAL); @@ -102,7 +102,7 @@ protected function create_local_file($content = 'test content') { $itemid = 0; $filepath = '/'; $sourcefield = 'Copyright stuff'; - $filerecord = array( + $filerecord = [ 'contextid' => $syscontext->id, 'component' => $component, 'filearea' => $filearea, @@ -111,7 +111,7 @@ protected function create_local_file($content = 'test content') { 'filename' => md5($content), // Unqiue content should guarentee unique path. 'source' => $sourcefield, 'mimetype' => 'text', - ); + ]; $file = $fs->create_file_from_string($filerecord, $content); manager::update_object_by_hash($file->get_contenthash(), OBJECT_LOCATION_LOCAL); @@ -331,7 +331,7 @@ protected function acquire_object_lock($filehash, $timeout = 0) { */ protected function delete_draft_files($contenthash) { global $DB; - $DB->delete_records('files', array('contenthash' => $contenthash)); + $DB->delete_records('files', ['contenthash' => $contenthash]); } /** diff --git a/db/tasks.php b/db/tasks.php index 5125d734..ac98e3ff 100644 --- a/db/tasks.php +++ b/db/tasks.php @@ -25,87 +25,87 @@ defined('MOODLE_INTERNAL') || die(); -$tasks = array( - array( +$tasks = [ + [ 'classname' => 'tool_objectfs\task\push_objects_to_storage', 'blocking' => 0, 'minute' => '*', 'hour' => '*', 'day' => '*', 'dayofweek' => '*', - 'month' => '*' - ), - array( + 'month' => '*', + ], + [ 'classname' => 'tool_objectfs\task\generate_status_report', 'blocking' => 0, 'minute' => '17', 'hour' => '*', 'day' => '*', 'dayofweek' => '*', - 'month' => '*' - ), - array( + 'month' => '*', + ], + [ 'classname' => 'tool_objectfs\task\delete_local_objects', 'blocking' => 0, 'minute' => '*', 'hour' => '*', 'day' => '*', 'dayofweek' => '*', - 'month' => '*' - ), - array( + 'month' => '*', + ], + [ 'classname' => 'tool_objectfs\task\orphan_objects', 'blocking' => 0, 'minute' => 'R', 'hour' => 'R', 'day' => '*', 'dayofweek' => '*', - 'month' => '*' - ), - array( + 'month' => '*', + ], + [ 'classname' => 'tool_objectfs\task\delete_orphaned_object_metadata', 'blocking' => 0, 'minute' => 'R', 'hour' => 'R', 'day' => '*', 'dayofweek' => '*', - 'month' => '*' - ), - array( + 'month' => '*', + ], + [ 'classname' => 'tool_objectfs\task\delete_local_empty_directories', 'blocking' => 0, 'minute' => '0', 'hour' => '1', 'day' => '*', 'dayofweek' => '*', - 'month' => '*' - ), - array( + 'month' => '*', + ], + [ 'classname' => 'tool_objectfs\task\pull_objects_from_storage', 'blocking' => 0, 'minute' => '*', 'hour' => '*', 'day' => '*', 'dayofweek' => '*', - 'month' => '*' - ), - array( + 'month' => '*', + ], + [ 'classname' => 'tool_objectfs\task\recover_error_objects', 'blocking' => 0, 'minute' => '34', 'hour' => '*/12', 'day' => '*', 'dayofweek' => '*', - 'month' => '*' - ), - array( + 'month' => '*', + ], + [ 'classname' => 'tool_objectfs\task\check_objects_location', 'blocking' => 0, 'minute' => 'R', 'hour' => '*', 'day' => '*', 'dayofweek' => '*', - 'month' => '*' - ), -); + 'month' => '*', + ], +]; diff --git a/db/upgrade.php b/db/upgrade.php index d9f697fe..19c70089 100644 --- a/db/upgrade.php +++ b/db/upgrade.php @@ -51,7 +51,7 @@ function xmldb_tool_objectfs_upgrade($oldversion) { if ($oldversion < 2017031000) { $table = new xmldb_table('tool_objectfs_objects'); - $key = new xmldb_key('contenthash', XMLDB_KEY_UNIQUE, array('contenthash')); + $key = new xmldb_key('contenthash', XMLDB_KEY_UNIQUE, ['contenthash']); $dbman->add_key($table, $key); upgrade_plugin_savepoint(true, 2017031000, 'tool', 'objectfs'); @@ -111,7 +111,7 @@ function xmldb_tool_objectfs_upgrade($oldversion) { $table = new xmldb_table('tool_objectfs_reports'); $table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE); $table->add_field('reportdate', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL); - $table->add_key('primary', XMLDB_KEY_PRIMARY, array('id')); + $table->add_key('primary', XMLDB_KEY_PRIMARY, ['id']); $table->add_index('reportdate_idx', XMLDB_INDEX_NOTUNIQUE, ['reportdate']); if (!$dbman->table_exists($table)) { $dbman->create_table($table); @@ -124,7 +124,7 @@ function xmldb_tool_objectfs_upgrade($oldversion) { $table->add_field('datakey', XMLDB_TYPE_CHAR, 15, null, XMLDB_NOTNULL); $table->add_field('objectcount', XMLDB_TYPE_INTEGER, 15, null, XMLDB_NOTNULL); $table->add_field('objectsum', XMLDB_TYPE_INTEGER, 20, null, XMLDB_NOTNULL); - $table->add_key('primary', XMLDB_KEY_PRIMARY, array('id')); + $table->add_key('primary', XMLDB_KEY_PRIMARY, ['id']); $table->add_index('reporttype_idx', XMLDB_INDEX_NOTUNIQUE, ['reporttype']); $table->add_index('reportid_idx', XMLDB_INDEX_NOTUNIQUE, ['reportid']); if (!$dbman->table_exists($table)) { diff --git a/lib.php b/lib.php index 6402c1fa..dd50df39 100644 --- a/lib.php +++ b/lib.php @@ -103,7 +103,7 @@ function tool_objectfs_pluginfile($course, $cm, context $context, $filearea, arr function tool_objectfs_status_checks() { if (get_config('tool_objectfs', 'proxyrangerequests')) { return [ - new tool_objectfs\check\proxy_range_request() + new tool_objectfs\check\proxy_range_request(), ]; } diff --git a/object_location.php b/object_location.php index aba1a4fd..8a9ab79c 100644 --- a/object_location.php +++ b/object_location.php @@ -31,7 +31,7 @@ admin_externalpage_setup('tool_objectfs_object_location_history'); $logformat = optional_param('download', '', PARAM_ALPHA); -$params = array(); +$params = []; if ($logformat) { $params['download'] = $logformat; } diff --git a/renderer.php b/renderer.php index 5fa571fb..9db9d07d 100644 --- a/renderer.php +++ b/renderer.php @@ -79,14 +79,14 @@ public function presignedurl_tests_load_files($fs) { $itemid = 0; $filepath = '/'; - $filerecord = array( + $filerecord = [ 'contextid' => $contextid, 'component' => $component, 'filearea' => $filearea, 'itemid' => $itemid, 'filepath' => $filepath, - 'filename' => $testfilename - ); + 'filename' => $testfilename, + ]; $testfile = $filestorage->get_file($contextid, $component, $filearea, $itemid, $filepath, $testfilename); if (!$testfile) { @@ -236,15 +236,15 @@ public function object_status_history_page_header($reports, $reportid) { $baseurl = '/admin/tool/objectfs/object_status.php'; - $previd = array(); - $nextid = array(); - $prevdisabled = array('disabled' => true); - $nextdisabled = array('disabled' => true); + $previd = []; + $nextid = []; + $prevdisabled = ['disabled' => true]; + $nextdisabled = ['disabled' => true]; end($reports); - $oldestid = array('reportid' => key($reports)); + $oldestid = ['reportid' => key($reports)]; reset($reports); - $latestid = array('reportid' => key($reports)); + $latestid = ['reportid' => key($reports)]; while ($reportid != key($reports)) { next($reports); @@ -252,7 +252,7 @@ public function object_status_history_page_header($reports, $reportid) { if (next($reports)) { $previd = ['reportid' => key($reports)]; - $prevdisabled = array(); + $prevdisabled = []; prev($reports); } else { end($reports); @@ -260,7 +260,7 @@ public function object_status_history_page_header($reports, $reportid) { if (prev($reports)) { $nextid = ['reportid' => key($reports)]; - $nextdisabled = array(); + $nextdisabled = []; next($reports); } else { reset($reports); diff --git a/tests/local/manager_test.php b/tests/local/manager_test.php index 608465d2..a195a710 100644 --- a/tests/local/manager_test.php +++ b/tests/local/manager_test.php @@ -38,7 +38,7 @@ class manager_test extends \tool_objectfs\tests\testcase { * * @return array */ - public function all_extensions_whitelisted_provider() { + public static function all_extensions_whitelisted_provider(): array { return [ [null, false], ['', false], @@ -69,7 +69,7 @@ public function test_all_extensions_whitelisted($signingwhitelist, $result) { * * @return array */ - public function is_extension_whitelisted_provider() { + public static function is_extension_whitelisted_provider(): array { return [ [null, 'file.tar', false], ['', 'file.tar', false], @@ -103,7 +103,7 @@ public function test_is_extension_whitelisted($signingwhitelist, $filename, $res * * @return array */ - public function get_header_provider() { + public static function get_header_provider(): array { return [ [[], '', ''], [[], 'Missing header', ''], diff --git a/tests/local/object_manipulator/checker_test.php b/tests/local/object_manipulator/checker_test.php index 5f24de71..7ac8f98d 100644 --- a/tests/local/object_manipulator/checker_test.php +++ b/tests/local/object_manipulator/checker_test.php @@ -48,7 +48,7 @@ protected function tearDown(): void { public function test_checker_get_location_local_if_object_is_local() { global $DB; $file = $this->create_local_object(); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $file->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $file->contenthash]); $this->assertEquals('string', gettype($location)); $this->assertEquals(OBJECT_LOCATION_LOCAL, $location); } @@ -56,7 +56,7 @@ public function test_checker_get_location_local_if_object_is_local() { public function test_checker_get_location_duplicated_if_object_is_duplicated() { global $DB; $file = $this->create_duplicated_object(); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $file->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $file->contenthash]); $this->assertEquals('string', gettype($location)); $this->assertEquals(OBJECT_LOCATION_DUPLICATED, $location); } @@ -64,7 +64,7 @@ public function test_checker_get_location_duplicated_if_object_is_duplicated() { public function test_checker_get_location_external_if_object_is_external() { global $DB; $file = $this->create_remote_object(); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $file->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $file->contenthash]); $this->assertEquals('string', gettype($location)); $this->assertEquals(OBJECT_LOCATION_EXTERNAL, $location); } @@ -82,7 +82,7 @@ public function test_checker_get_candidate_objects_will_not_get_objects() { public function test_checker_get_candidate_objects_will_get_object() { global $DB; $localobject = $this->create_local_object('test_checker_get_candidate_objects_will_get_object'); - $DB->delete_records('tool_objectfs_objects', array('contenthash' => $localobject->contenthash)); + $DB->delete_records('tool_objectfs_objects', ['contenthash' => $localobject->contenthash]); self::assertTrue($this->objects_contain_hash($localobject->contenthash)); } @@ -104,27 +104,27 @@ public function test_checker_manipulate_object_method_will_get_correct_location_ $file = $this->create_local_object(); $reflection = new \ReflectionMethod(checker::class, "manipulate_object"); $reflection->setAccessible(true); - $this->assertEquals(OBJECT_LOCATION_LOCAL, $reflection->invokeArgs($this->checker, array($file))); + $this->assertEquals(OBJECT_LOCATION_LOCAL, $reflection->invokeArgs($this->checker, [$file])); } public function test_checker_manipulate_object_method_will_get_correct_location_if_file_is_duplicated() { $file = $this->create_duplicated_object(); $reflection = new \ReflectionMethod(checker::class, "manipulate_object"); $reflection->setAccessible(true); - $this->assertEquals(OBJECT_LOCATION_DUPLICATED, $reflection->invokeArgs($this->checker, array($file))); + $this->assertEquals(OBJECT_LOCATION_DUPLICATED, $reflection->invokeArgs($this->checker, [$file])); } public function test_checker_manipulate_object_method_will_get_correct_location_if_file_is_external() { $file = $this->create_remote_object(); $reflection = new \ReflectionMethod(checker::class, "manipulate_object"); $reflection->setAccessible(true); - $this->assertEquals(OBJECT_LOCATION_EXTERNAL, $reflection->invokeArgs($this->checker, array($file))); + $this->assertEquals(OBJECT_LOCATION_EXTERNAL, $reflection->invokeArgs($this->checker, [$file])); } public function test_checker_manipulate_object_method_will_get_error_location_on_error_file() { $file = $this->create_error_object(); $reflection = new \ReflectionMethod(checker::class, "manipulate_object"); $reflection->setAccessible(true); - $this->assertEquals(OBJECT_LOCATION_ERROR, $reflection->invokeArgs($this->checker, array($file))); + $this->assertEquals(OBJECT_LOCATION_ERROR, $reflection->invokeArgs($this->checker, [$file])); } } diff --git a/tests/local/object_manipulator/deleter_test.php b/tests/local/object_manipulator/deleter_test.php index a29499f8..56ddfa5b 100644 --- a/tests/local/object_manipulator/deleter_test.php +++ b/tests/local/object_manipulator/deleter_test.php @@ -87,9 +87,9 @@ public function test_deleter_can_delete_object() { global $DB; $object = $this->create_duplicated_object(); - $this->deleter->execute(array($object)); + $this->deleter->execute([$object]); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_EXTERNAL, $location); $this->assertFalse($this->is_locally_readable_by_hash($object->contenthash)); $this->assertTrue($this->is_externally_readable_by_hash($object->contenthash)); @@ -99,9 +99,9 @@ public function test_deleter_can_handle_local_object() { global $DB; $object = $this->create_local_object(); - $this->deleter->execute(array($object)); + $this->deleter->execute([$object]); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_LOCAL, $location); $this->assertTrue($this->is_locally_readable_by_hash($object->contenthash)); $this->assertFalse($this->is_externally_readable_by_hash($object->contenthash)); @@ -111,9 +111,9 @@ public function test_deleter_can_handle_remote_object() { global $DB; $object = $this->create_remote_object(); - $this->deleter->execute(array($object)); + $this->deleter->execute([$object]); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_EXTERNAL, $location); $this->assertFalse($this->is_locally_readable_by_hash($object->contenthash)); $this->assertTrue($this->is_externally_readable_by_hash($object->contenthash)); @@ -124,9 +124,9 @@ public function test_deleter_will_delete_no_objects_if_deletelocal_disabled() { $object = $this->create_duplicated_object(); $this->set_deleter_config('deletelocal', 0); - $this->deleter->execute(array($object)); + $this->deleter->execute([$object]); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_DUPLICATED, $location); $this->assertTrue($this->is_locally_readable_by_hash($object->contenthash)); $this->assertTrue($this->is_externally_readable_by_hash($object->contenthash)); @@ -134,7 +134,7 @@ public function test_deleter_will_delete_no_objects_if_deletelocal_disabled() { public function test_deleter_can_delete_multiple_objects() { global $DB; - $objects = array(); + $objects = []; for ($i = 0; $i < 5; $i++) { $objects[] = $this->create_duplicated_object("Object $i"); } @@ -142,7 +142,7 @@ public function test_deleter_can_delete_multiple_objects() { $this->deleter->execute($objects); foreach ($objects as $object) { - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_EXTERNAL, $location); $this->assertFalse($this->is_locally_readable_by_hash($object->contenthash)); $this->assertTrue($this->is_externally_readable_by_hash($object->contenthash)); diff --git a/tests/local/object_manipulator/orphaner_test.php b/tests/local/object_manipulator/orphaner_test.php index 6befdf30..1498e0ff 100644 --- a/tests/local/object_manipulator/orphaner_test.php +++ b/tests/local/object_manipulator/orphaner_test.php @@ -96,7 +96,7 @@ public function test_orphaner_finds_correct_candidates() { // Update that object to have a different hash, to mock a non-existent // mdl_file with an objectfs record (orphaned). - $DB->set_field('files', 'contenthash', 'different', array('contenthash' => $object->contenthash)); + $DB->set_field('files', 'contenthash', 'different', ['contenthash' => $object->contenthash]); // Expect one candidate - no matching contenthash in {files}. $objects = $finder->get(); diff --git a/tests/local/object_manipulator/puller_test.php b/tests/local/object_manipulator/puller_test.php index 0c793613..d54171cb 100644 --- a/tests/local/object_manipulator/puller_test.php +++ b/tests/local/object_manipulator/puller_test.php @@ -77,7 +77,7 @@ public function test_puller_get_candidate_objects_will_not_get_duplicated_or_loc public function test_puller_get_candidate_objects_will_not_get_objects_over_sizethreshold() { global $DB; $remoteobject = $this->create_remote_object(); - $DB->set_field('files', 'filesize', 10, array('contenthash' => $remoteobject->contenthash)); + $DB->set_field('files', 'filesize', 10, ['contenthash' => $remoteobject->contenthash]); $this->set_puller_config('sizethreshold', 0); self::assertFalse($this->objects_contain_hash($remoteobject->contenthash)); @@ -87,9 +87,9 @@ public function test_puller_can_pull_remote_file() { global $DB; $object = $this->create_remote_object(); - $this->puller->execute(array($object)); + $this->puller->execute([$object]); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_DUPLICATED, $location); $this->assertTrue($this->is_locally_readable_by_hash($object->contenthash)); $this->assertTrue($this->is_externally_readable_by_hash($object->contenthash)); @@ -99,9 +99,9 @@ public function test_puller_can_handle_duplicated_file() { global $DB; $object = $this->create_duplicated_object(); - $this->puller->execute(array($object)); + $this->puller->execute([$object]); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_DUPLICATED, $location); $this->assertTrue($this->is_locally_readable_by_hash($object->contenthash)); $this->assertTrue($this->is_externally_readable_by_hash($object->contenthash)); @@ -111,9 +111,9 @@ public function test_puller_can_handle_local_file() { global $DB; $object = $this->create_local_object(); - $this->puller->execute(array($object)); + $this->puller->execute([$object]); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_LOCAL, $location); $this->assertTrue($this->is_locally_readable_by_hash($object->contenthash)); $this->assertFalse($this->is_externally_readable_by_hash($object->contenthash)); diff --git a/tests/local/object_manipulator/pusher_test.php b/tests/local/object_manipulator/pusher_test.php index 3c1c93a1..e902f0be 100644 --- a/tests/local/object_manipulator/pusher_test.php +++ b/tests/local/object_manipulator/pusher_test.php @@ -80,7 +80,7 @@ public function test_pusher_get_candidate_objects_wont_get_objects_bigger_than_m global $DB; $object = $this->create_local_object(); $maximumfilesize = $this->filesystem->get_maximum_upload_filesize() + 1; - $DB->set_field('tool_objectfs_objects', 'filesize', $maximumfilesize, array('contenthash' => $object->contenthash)); + $DB->set_field('tool_objectfs_objects', 'filesize', $maximumfilesize, ['contenthash' => $object->contenthash]); self::assertFalse($this->objects_contain_hash($object->contenthash)); } @@ -89,7 +89,7 @@ public function test_pusher_get_candidate_objects_wont_get_objects_under_size_th global $DB; $this->set_pusher_config('sizethreshold', 100); $object = $this->create_local_object(); - $DB->set_field('tool_objectfs_objects', 'filesize', 10, array('contenthash' => $object->contenthash)); + $DB->set_field('tool_objectfs_objects', 'filesize', 10, ['contenthash' => $object->contenthash]); self::assertFalse($this->objects_contain_hash($object->contenthash)); } @@ -105,9 +105,9 @@ public function test_pusher_can_push_local_file() { global $DB; $object = $this->create_local_object(); - $this->pusher->execute(array($object)); + $this->pusher->execute([$object]); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_DUPLICATED, $location); $this->assertTrue($this->is_locally_readable_by_hash($object->contenthash)); $this->assertTrue($this->is_externally_readable_by_hash($object->contenthash)); @@ -117,9 +117,9 @@ public function test_pusher_can_handle_duplicated_file() { global $DB; $object = $this->create_duplicated_object(); - $this->pusher->execute(array($object)); + $this->pusher->execute([$object]); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_DUPLICATED, $location); $this->assertTrue($this->is_locally_readable_by_hash($object->contenthash)); $this->assertTrue($this->is_externally_readable_by_hash($object->contenthash)); @@ -129,9 +129,9 @@ public function test_pusher_can_handle_remote_file() { global $DB; $object = $this->create_remote_object(); - $this->pusher->execute(array($object)); + $this->pusher->execute([$object]); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_EXTERNAL, $location); $this->assertFalse($this->is_locally_readable_by_hash($object->contenthash)); $this->assertTrue($this->is_externally_readable_by_hash($object->contenthash)); @@ -139,7 +139,7 @@ public function test_pusher_can_handle_remote_file() { public function test_pusher_can_push_multiple_objects() { global $DB; - $objects = array(); + $objects = []; for ($i = 0; $i < 5; $i++) { $objects[] = $this->create_local_object("Object $i"); } @@ -147,7 +147,7 @@ public function test_pusher_can_push_multiple_objects() { $this->pusher->execute($objects); foreach ($objects as $object) { - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_DUPLICATED, $location); $this->assertTrue($this->is_locally_readable_by_hash($object->contenthash)); $this->assertTrue($this->is_externally_readable_by_hash($object->contenthash)); @@ -164,7 +164,7 @@ public function test_get_candidate_objects_get_one_object_if_files_have_same_has $this->pusher->execute($objects); $object = $this->create_local_object(); - $file = $DB->get_record('files', array('contenthash' => $object->contenthash)); + $file = $DB->get_record('files', ['contenthash' => $object->contenthash]); // Update mimetype to something different and insert as new file. $file->mimetype = "differentMimeType"; diff --git a/tests/local/object_manipulator/recoverer_test.php b/tests/local/object_manipulator/recoverer_test.php index 9a823555..1b8f7927 100644 --- a/tests/local/object_manipulator/recoverer_test.php +++ b/tests/local/object_manipulator/recoverer_test.php @@ -59,44 +59,44 @@ public function test_recoverer_get_candidate_objects_will_get_error_objects() { public function test_recoverer_will_recover_local_objects() { global $DB; $object = $this->create_local_object(); - $DB->set_field('tool_objectfs_objects', 'location', OBJECT_LOCATION_ERROR, array('contenthash' => $object->contenthash)); + $DB->set_field('tool_objectfs_objects', 'location', OBJECT_LOCATION_ERROR, ['contenthash' => $object->contenthash]); - $this->recoverer->execute(array($object)); + $this->recoverer->execute([$object]); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_LOCAL, $location); } public function test_recoverer_will_recover_duplicated_objects() { global $DB; $object = $this->create_duplicated_object(); - $DB->set_field('tool_objectfs_objects', 'location', OBJECT_LOCATION_ERROR, array('contenthash' => $object->contenthash)); + $DB->set_field('tool_objectfs_objects', 'location', OBJECT_LOCATION_ERROR, ['contenthash' => $object->contenthash]); - $this->recoverer->execute(array($object)); + $this->recoverer->execute([$object]); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_DUPLICATED, $location); } public function test_recoverer_will_recover_remote_objects() { global $DB; $object = $this->create_remote_object(); - $DB->set_field('tool_objectfs_objects', 'location', OBJECT_LOCATION_ERROR, array('contenthash' => $object->contenthash)); + $DB->set_field('tool_objectfs_objects', 'location', OBJECT_LOCATION_ERROR, ['contenthash' => $object->contenthash]); - $this->recoverer->execute(array($object)); + $this->recoverer->execute([$object]); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_EXTERNAL, $location); } public function test_recoverer_will_not_recover_error_objects() { global $DB; $object = $this->create_error_object(); - $DB->set_field('tool_objectfs_objects', 'location', OBJECT_LOCATION_ERROR, array('contenthash' => $object->contenthash)); + $DB->set_field('tool_objectfs_objects', 'location', OBJECT_LOCATION_ERROR, ['contenthash' => $object->contenthash]); - $this->recoverer->execute(array($object)); + $this->recoverer->execute([$object]); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $object->contenthash)); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $object->contenthash]); $this->assertEquals(OBJECT_LOCATION_ERROR, $location); } } diff --git a/tests/local/report/object_status_test.php b/tests/local/report/object_status_test.php index 6aa8ee1e..bbc895d4 100644 --- a/tests/local/report/object_status_test.php +++ b/tests/local/report/object_status_test.php @@ -168,7 +168,7 @@ public function test_cleanup_reports() { * * @return array */ - public function object_status_add_barchart_method_provider() { + public static function object_status_add_barchart_method_provider(): array { return [ [0, 0, '', 0, '0'], [0, 100, 'count', 0, '
' . number_format(0) . '
'], @@ -212,7 +212,7 @@ public function test_object_status_add_barchart_method($value, $max, $type, $pre * * @return array */ - public function object_status_get_size_range_from_logsize_provider() { + public static function object_status_get_size_range_from_logsize_provider(): array { return [ ['1', '< ' . display_size(1024)], ['10', display_size(1024) . ' - ' . display_size(2048)], diff --git a/tests/local/store/clients_test.php b/tests/local/store/clients_test.php index 079327e2..6403a234 100644 --- a/tests/local/store/clients_test.php +++ b/tests/local/store/clients_test.php @@ -16,8 +16,8 @@ namespace tool_objectfs\local\store; -use \tool_objectfs\tests\test_digitalocean_integration_client as digitaloceanclient; -use \tool_objectfs\tests\test_s3_integration_client as s3client; +use tool_objectfs\tests\test_digitalocean_integration_client as digitaloceanclient; +use tool_objectfs\tests\test_s3_integration_client as s3client; /** * Client tests. @@ -32,7 +32,7 @@ class clients_test extends \advanced_testcase { * * @return \array[][] */ - public function s3_client_test_connection_if_not_configured_properly_data_provider() { + public static function s3_client_test_connection_if_not_configured_properly_data_provider(): array { return [ [[]], [['s3_bucket' => '', 's3_region' => 'test', 's3_usesdkcreds' => 0, 's3_key' => 'test', 's3_secret' => 'test']], @@ -72,7 +72,7 @@ public function test_s3_client_test_connection_if_not_configured_properly(array * * @return \array[][] */ - public function digitalocean_client_test_connection_if_not_configured_properly_data_provider() { + public static function digitalocean_client_test_connection_if_not_configured_properly_data_provider(): array { return [ [[]], [['do_key' => '', 'do_secret' => '', 'do_region' => '']], diff --git a/tests/object_file_system_test.php b/tests/object_file_system_test.php index 428db219..0ccd2a5a 100644 --- a/tests/object_file_system_test.php +++ b/tests/object_file_system_test.php @@ -234,7 +234,7 @@ public function test_delete_object_from_local_by_hash_if_can_verify_external_obj * delete_empty_folders_provider * @return array */ - public function delete_empty_folders_provider() { + public static function delete_empty_folders_provider(): array { return [ [ /* @@ -334,11 +334,11 @@ public function test_readfile_updates_object_with_error_location_on_fail() { // Phpunit will fail if PHP warning is thrown (which we want) // so we surpress here. - set_error_handler(array($this, 'error_surpressor')); + set_error_handler([$this, 'error_surpressor']); $this->filesystem->readfile($fakefile); restore_error_handler(); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $fakefile->get_contenthash())); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $fakefile->get_contenthash()]); $this->assertEquals(OBJECT_LOCATION_ERROR, $location); } @@ -366,11 +366,11 @@ public function test_get_content_updates_object_with_error_location_on_fail() { // Phpunit will fail if PHP warning is thrown (which we want) // so we surpress here. - set_error_handler(array($this, 'error_surpressor')); + set_error_handler([$this, 'error_surpressor']); $this->filesystem->get_content($fakefile); restore_error_handler(); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $fakefile->get_contenthash())); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $fakefile->get_contenthash()]); $this->assertEquals(OBJECT_LOCATION_ERROR, $location); } @@ -389,11 +389,11 @@ public function test_xsendfile_updates_object_with_error_location_on_fail() { // Phpunit will fail if PHP warning is thrown (which we want) // so we surpress here. - set_error_handler(array($this, 'error_surpressor')); + set_error_handler([$this, 'error_surpressor']); $this->filesystem->xsendfile($fakefile->get_contenthash()); restore_error_handler(); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $fakefile->get_contenthash())); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $fakefile->get_contenthash()]); $this->assertEquals(OBJECT_LOCATION_ERROR, $location); } @@ -429,11 +429,11 @@ public function test_get_content_file_handle_updates_object_with_error_location_ // Phpunit will fail if PHP warning is thrown (which we want) // so we surpress here. - set_error_handler(array($this, 'error_surpressor')); + set_error_handler([$this, 'error_surpressor']); $filehandle = $this->filesystem->get_content_file_handle($fakefile); restore_error_handler(); - $location = $DB->get_field('tool_objectfs_objects', 'location', array('contenthash' => $fakefile->get_contenthash())); + $location = $DB->get_field('tool_objectfs_objects', 'location', ['contenthash' => $fakefile->get_contenthash()]); $this->assertEquals(OBJECT_LOCATION_ERROR, $location); } @@ -443,7 +443,7 @@ public function test_remove_file_will_remove_local_file() { $filehash = $file->get_contenthash(); // Delete file record so remove file will remove. - $DB->delete_records('files', array('contenthash' => $filehash)); + $DB->delete_records('files', ['contenthash' => $filehash]); $this->filesystem->remove_file($filehash); $islocalreadable = $this->filesystem->is_file_readable_locally_by_hash($filehash); @@ -456,7 +456,7 @@ public function test_remove_file_will_not_remove_remote_file() { $filehash = $file->get_contenthash(); // Delete file record so remove file will remove. - $DB->delete_records('files', array('contenthash' => $filehash)); + $DB->delete_records('files', ['contenthash' => $filehash]); $this->filesystem->remove_file($filehash); $isremotereadable = $this->is_externally_readable_by_hash($filehash); @@ -654,40 +654,40 @@ public function test_presigned_url_configured_method_returns_true_if_configured( * presigned_url_should_redirect_provider * @return array */ - public function presigned_url_should_redirect_provider() { - $provider = array(); + public static function presigned_url_should_redirect_provider(): array { + $provider = []; // Testing defaults. - $provider[] = array('Default', 'Default', false); + $provider[] = ['Default', 'Default', false]; // Testing $enablepresignedurls. - $provider[] = array(1, 'Default', true); - $provider[] = array('1', 'Default', true); - $provider[] = array(0, 'Default', false); - $provider[] = array('0', 'Default', false); - $provider[] = array('', 'Default', false); - $provider[] = array(null, 'Default', false); + $provider[] = [1, 'Default', true]; + $provider[] = ['1', 'Default', true]; + $provider[] = [0, 'Default', false]; + $provider[] = ['0', 'Default', false]; + $provider[] = ['', 'Default', false]; + $provider[] = [null, 'Default', false]; // Testing $presignedminfilesize. - $provider[] = array(1, 0, true); - $provider[] = array(1, '0', true); - $provider[] = array(1, '', true); + $provider[] = [1, 0, true]; + $provider[] = [1, '0', true]; + $provider[] = [1, '', true]; // Testing minimum file size to be greater than file size. // 12 is a size of the file with 'test content' content. - $provider[] = array(1, 13, false); - $provider[] = array(1, '13', false); + $provider[] = [1, 13, false]; + $provider[] = [1, '13', false]; // Testing minimum file size to be less than file size. // 12 is a size of the file with 'test content' content. - $provider[] = array(1, 11, true); - $provider[] = array(1, '11', true); + $provider[] = [1, 11, true]; + $provider[] = [1, '11', true]; // Testing nulls and empty strings. - $provider[] = array(null, null, false); - $provider[] = array(null, '', false); - $provider[] = array('', null, false); - $provider[] = array('', '', false); + $provider[] = [null, null, false]; + $provider[] = [null, '', false]; + $provider[] = ['', null, false]; + $provider[] = ['', '', false]; return $provider; } @@ -733,7 +733,7 @@ public function test_presigned_url_should_redirect_method_with_data_provider($en * * @return array */ - public function get_expiration_time_method_if_supported_provider() { + public static function get_expiration_time_method_if_supported_provider(): array { $now = time(); // Seconds after the minute from X. @@ -840,7 +840,7 @@ public function test_get_filesize_by_contenthash() { * * @return array */ - public function get_valid_http_ranges_provider() { + public static function get_valid_http_ranges_provider(): array { return [ ['', 0, false], ['bytes=0-', 100, (object)['rangefrom' => 0, 'rangeto' => 99, 'length' => 100]], @@ -870,7 +870,7 @@ public function test_get_valid_http_ranges($httprangeheader, $filesize, $expecte * * @return array */ - public function curl_range_request_to_presigned_url_provider() { + public static function curl_range_request_to_presigned_url_provider(): array { return [ ['15-bytes string', (object)['rangefrom' => 0, 'rangeto' => 14, 'length' => 15], '15-bytes string'], ['15-bytes string', (object)['rangefrom' => 0, 'rangeto' => 9, 'length' => 10], '15-bytes s'], diff --git a/tests/task/populate_objects_filesize_test.php b/tests/task/populate_objects_filesize_test.php index e5fb4c19..9ff08ee0 100644 --- a/tests/task/populate_objects_filesize_test.php +++ b/tests/task/populate_objects_filesize_test.php @@ -44,7 +44,7 @@ public function test_empty_filesizes_updated() { $this->create_local_file("Test 2")->get_contenthash(), $this->create_local_file("Test 3")->get_contenthash(), $this->create_local_file("Test 4")->get_contenthash(), - $this->create_local_file("This is a looong name")->get_contenthash() + $this->create_local_file("This is a looong name")->get_contenthash(), ]; // Set all objects to have a filesize of null. @@ -131,7 +131,7 @@ public function test_that_non_null_values_are_not_updated() { $this->create_local_file("Test 2")->get_contenthash(), $this->create_local_file("Test 3")->get_contenthash(), $this->create_local_file("Test 4")->get_contenthash(), - $this->create_local_file("This is a looong name")->get_contenthash() + $this->create_local_file("This is a looong name")->get_contenthash(), ]; // Set all objects to have a filesize of null. diff --git a/version.php b/version.php index f53c4dc5..bea5f7e8 100644 --- a/version.php +++ b/version.php @@ -27,7 +27,7 @@ $plugin->version = 2023051701; // The current plugin version (Date: YYYYMMDDXX). $plugin->release = 2023051701; // Same as version. -$plugin->requires = 2023042400; // Requires 4.2 +$plugin->requires = 2023042400; // Requires 4.2. $plugin->component = "tool_objectfs"; $plugin->maturity = MATURITY_STABLE; $plugin->supported = [402, 405]; From 34216637baf0965363489d19ec1ff5f8cb489e2b Mon Sep 17 00:00:00 2001 From: sam marshall Date: Mon, 19 Feb 2024 15:50:34 +0000 Subject: [PATCH 07/28] Remove use of utf8_encode (deprecated in PHP 8.2) --- classes/local/store/s3/client.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/classes/local/store/s3/client.php b/classes/local/store/s3/client.php index a3cce110..a6e2598e 100644 --- a/classes/local/store/s3/client.php +++ b/classes/local/store/s3/client.php @@ -670,7 +670,13 @@ private function get_nice_filename($headers) { if (!empty($originalfilename)) { $result['Content-Disposition'] = $contentdisposition; - $result['filename'] = 'filename="' . utf8_encode($originalfilename) . '"'; + // The filename parameter must be in ISO-8859-1, however it works in browsers if + // you treat the original UTF-8 string as ISO-8859-1 characters. To achieve that + // here, we encode the UTF-8 as if it were ISO-8859-1. This behaviour is hideous + // so it would be nice to use the optional filename* field (RFC 5987) but S3 still + // complains if we do that. + $jankyfilename = \core_text::convert($originalfilename, 'ISO-8859-1'); + $result['filename'] = 'filename="' . $jankyfilename . '"'; $result['Content-Type'] = $originalcontenttype; } } From 08c0ecfd77f5694dc9a03770bccfcd866b350912 Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Tue, 30 Jul 2024 09:26:01 +1000 Subject: [PATCH 08/28] chore: remove legacy polyfill --- classes/privacy/provider.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/classes/privacy/provider.php b/classes/privacy/provider.php index ff971d00..2e6d16c6 100644 --- a/classes/privacy/provider.php +++ b/classes/privacy/provider.php @@ -23,13 +23,11 @@ */ namespace tool_objectfs\privacy; use core_privacy\local\metadata\null_provider; -use core_privacy\local\legacy_polyfill; /** * Class provider * @package tool_objectfs */ class provider implements null_provider { - use legacy_polyfill; /** * Get the language string identifier with the component's language * file to explain why this plugin stores no data. From 374e48e90f8385b3cf169a0cab00f3c152870ce4 Mon Sep 17 00:00:00 2001 From: guillermogomez Date: Wed, 22 May 2024 21:29:34 -0400 Subject: [PATCH 09/28] Issue #609: When a file size is 0 bytes do not mark it as location error --- classes/local/store/object_file_system.php | 2 +- classes/local/store/s3/file_system.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/classes/local/store/object_file_system.php b/classes/local/store/object_file_system.php index fb3da10d..de30f1d2 100644 --- a/classes/local/store/object_file_system.php +++ b/classes/local/store/object_file_system.php @@ -512,7 +512,7 @@ public function readfile(\stored_file $file) { $this->logger->log_object_read('readfile', $path, $file->get_filesize()); - if (!$success) { + if ($success === false) { manager::update_object_by_hash($file->get_contenthash(), OBJECT_LOCATION_ERROR); } } diff --git a/classes/local/store/s3/file_system.php b/classes/local/store/s3/file_system.php index dd911092..93637dd4 100644 --- a/classes/local/store/s3/file_system.php +++ b/classes/local/store/s3/file_system.php @@ -71,7 +71,7 @@ public function readfile(\stored_file $file) { $this->get_logger()->end_timing(); $this->get_logger()->log_object_read('readfile', $path, $file->get_filesize()); - if (!$success) { + if ($success === false) { manager::update_object_by_hash($file->get_contenthash(), OBJECT_LOCATION_ERROR); throw new \file_exception('storedfilecannotreadfile', $file->get_filename()); } From 8a2c72c1b972881fbfd552c094358adf5454ac3c Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Tue, 17 Sep 2024 14:32:25 +1000 Subject: [PATCH 10/28] bugfix: use test client in unit tests --- classes/local/manager.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/classes/local/manager.php b/classes/local/manager.php index 5d791f97..acd2b30e 100644 --- a/classes/local/manager.php +++ b/classes/local/manager.php @@ -329,6 +329,10 @@ public static function get_available_fs_list() { * @return string */ public static function get_client_classname_from_fs($filesystem) { + // Unit tests need to return the test client. + if ($filesystem == '\tool_objectfs\tests\test_file_system') { + return '\tool_objectfs\tests\test_client'; + } $clientclass = str_replace('_file_system', '', $filesystem); return str_replace('tool_objectfs\\', 'tool_objectfs\\local\\store\\', $clientclass.'\\client'); } From 5b7a4de8a898e6e1a83c66dcfad90160a941bc82 Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Tue, 17 Sep 2024 14:35:35 +1000 Subject: [PATCH 11/28] feat: add token expiry check --- classes/check/token_expiry.php | 71 ++++++++++++++++++++ classes/local/store/azure/client.php | 35 ++++++++++ classes/local/store/object_client.php | 6 ++ classes/local/store/object_client_base.php | 9 +++ classes/tests/test_client.php | 9 +++ lang/en/tool_objectfs.php | 6 ++ lib.php | 10 +-- settings.php | 4 ++ tests/local/store/clients_test.php | 41 ++++++++++++ tests/token_expiry_test.php | 77 ++++++++++++++++++++++ version.php | 4 +- 11 files changed, 266 insertions(+), 6 deletions(-) create mode 100644 classes/check/token_expiry.php create mode 100644 tests/token_expiry_test.php diff --git a/classes/check/token_expiry.php b/classes/check/token_expiry.php new file mode 100644 index 00000000..ad6e8d6d --- /dev/null +++ b/classes/check/token_expiry.php @@ -0,0 +1,71 @@ +. + +namespace tool_objectfs\check; + +use core\check\check; +use core\check\result; + +/** + * Token expiry check. + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class token_expiry extends check { + /** + * Checks the token expiry time against thresholds + * @return result + */ + public function get_result(): result { + $config = \tool_objectfs\local\manager::get_objectfs_config(); + $client = \tool_objectfs\local\manager::get_client($config); + + // No client set - n/a. + if (empty($client)) { + return new result(result::NA, get_string('check:tokenexpiry:na', 'tool_objectfs')); + } + + $expirytime = $client->get_token_expiry_time(); + $secondsleft = $expirytime - time(); + + $strparams = [ + 'dayssince' => abs(round($secondsleft / DAYSECS)), + 'time' => userdate($expirytime), + ]; + + // Not implemented or token not set - n/a. + if ($expirytime == -1) { + return new result(result::NA, get_string('check:tokenexpiry:na', 'tool_objectfs')); + } + + // Is in past - token has expired. + if ($secondsleft < 0) { + return new result(result::CRITICAL, get_string('check:tokenexpiry:expired', 'tool_objectfs', $strparams)); + } + + // Is in warning period - warn. + $warnthreshold = (int) $config->tokenexpirywarnperiod; + if ($secondsleft < $warnthreshold) { + return new result(result::WARNING, get_string('check:tokenexpiry:expiresin', 'tool_objectfs', $strparams)); + } + + // Else ok. + return new result(result::OK, get_string('check:tokenexpiry:expiresin', 'tool_objectfs', $strparams)); + } +} diff --git a/classes/local/store/azure/client.php b/classes/local/store/azure/client.php index c96bdde8..ed755e0f 100644 --- a/classes/local/store/azure/client.php +++ b/classes/local/store/azure/client.php @@ -25,8 +25,10 @@ namespace tool_objectfs\local\store\azure; +use admin_setting_description; use SimpleXMLElement; use stdClass; +use tool_objectfs\check\token_expiry; use tool_objectfs\local\store\azure\stream_wrapper; use tool_objectfs\local\store\object_client_base; @@ -360,9 +362,42 @@ public function define_client_section($settings, $config) { new \lang_string('settings:azure:sastoken', 'tool_objectfs'), new \lang_string('settings:azure:sastoken_help', 'tool_objectfs'), '')); + // Admin_setting_check only exists in 4.5+, in lower versions fallback to a basic description. + if (class_exists('admin_setting_check')) { + $settings->add(new admin_setting_check('tool_objectfs/check_tokenexpiry', new token_expiry(), true)); + } else { + $summary = (new token_expiry())->get_result()->get_summary(); + $settings->add(new admin_setting_description('tool_objectfs/tokenexpirycheckresult', + get_string('checktoken_expiry', 'tool_objectfs'), $summary)); + } + return $settings; } + /** + * Returns token expiry time + * @return int + */ + public function get_token_expiry_time(): int { + if (empty($this->config->azure_sastoken)) { + return -1; + } + + // Parse the sas token (it just uses url parameter encoding). + $parts = []; + parse_str($this->config->azure_sastoken, $parts); + + // Get the 'se' part (signed expiry). + if (!isset($parts['se'])) { + // Assume expired (malformed). + return 0; + } + + // Parse timestamp string into unix timestamp int. + $expirystr = $parts['se']; + return strtotime($expirystr); + } + /** * Extract an error code from the XML response. * diff --git a/classes/local/store/object_client.php b/classes/local/store/object_client.php index 9121a824..cd8cbc7f 100644 --- a/classes/local/store/object_client.php +++ b/classes/local/store/object_client.php @@ -137,6 +137,12 @@ public function proxy_range_request(\stored_file $file, $ranges); */ public function test_range_request($filesystem); + /** + * Get the expiry time of the token used for this fs. + * returns -1 if not implemented, or no token is set. + * @return int unix timestamp the token set expires at + */ + public function get_token_expiry_time(): int; } diff --git a/classes/local/store/object_client_base.php b/classes/local/store/object_client_base.php index 4f03418d..fa1b7e8f 100644 --- a/classes/local/store/object_client_base.php +++ b/classes/local/store/object_client_base.php @@ -187,4 +187,13 @@ public function test_connection() { public function test_permissions($testdelete) { return (object)['success' => false, 'details' => '']; } + + /** + * Return expiry time of token, default is -1 meaning not implemented/enabled. + * @return int + */ + public function get_token_expiry_time(): int { + // Returning -1 = not implemented. + return -1; + } } diff --git a/classes/tests/test_client.php b/classes/tests/test_client.php index 7a380c17..dbed5711 100644 --- a/classes/tests/test_client.php +++ b/classes/tests/test_client.php @@ -157,5 +157,14 @@ public function test_permissions($testdelete) { public function get_maximum_upload_size() { return $this->maxupload; } + + /** + * Returns test expiry time. + * @return int + */ + public function get_token_expiry_time(): int { + global $CFG; + return $CFG->objectfs_phpunit_token_expiry_time; + } } diff --git a/lang/en/tool_objectfs.php b/lang/en/tool_objectfs.php index e8f48876..74b383a5 100644 --- a/lang/en/tool_objectfs.php +++ b/lang/en/tool_objectfs.php @@ -269,3 +269,9 @@ $string['check:proxyrangerequestsdisabled'] = 'The proxy range request setting is disabled.'; $string['checkproxy_range_request'] = 'Pre-signed URL range request proxy'; + +$string['checktoken_expiry'] = 'Token expiry'; +$string['check:tokenexpiry:expiresin'] = 'Token expires in {$a->dayssince} days on {$a->time}'; +$string['check:tokenexpiry:expired'] = 'Token expired for {$a->dayssince} days. Expired on {$a->time}'; +$string['check:tokenexpiry:na'] = 'Token expired not implemented for filesystem, or no token is set'; +$string['settings:tokenexpirywarnperiod'] = 'Token expiry warn period'; diff --git a/lib.php b/lib.php index dd50df39..708063a8 100644 --- a/lib.php +++ b/lib.php @@ -101,11 +101,13 @@ function tool_objectfs_pluginfile($course, $cm, context $context, $filearea, arr * @return array */ function tool_objectfs_status_checks() { + $checks = [ + new tool_objectfs\check\token_expiry(), + ]; + if (get_config('tool_objectfs', 'proxyrangerequests')) { - return [ - new tool_objectfs\check\proxy_range_request(), - ]; + $checks[] = new tool_objectfs\check\proxy_range_request(); } - return []; + return $checks; } diff --git a/settings.php b/settings.php index a941e506..aef3c571 100644 --- a/settings.php +++ b/settings.php @@ -106,6 +106,10 @@ new lang_string('settings:useproxy_help', 'tool_objectfs'), 0)); + $settings->add(new admin_setting_configduration('tool_objectfs/tokenexpirywarnperiod', + new lang_string('settings:tokenexpirywarnperiod', 'tool_objectfs'), + '', 60 * DAYSECS, DAYSECS)); + $settings->add(new admin_setting_heading('tool_objectfs/filetransfersettings', new lang_string('settings:filetransferheader', 'tool_objectfs'), '')); diff --git a/tests/local/store/clients_test.php b/tests/local/store/clients_test.php index 6403a234..2fcec176 100644 --- a/tests/local/store/clients_test.php +++ b/tests/local/store/clients_test.php @@ -18,6 +18,7 @@ use tool_objectfs\tests\test_digitalocean_integration_client as digitaloceanclient; use tool_objectfs\tests\test_s3_integration_client as s3client; +use tool_objectfs\tests\test_azure_integration_client as azureclient; /** * Client tests. @@ -107,4 +108,44 @@ public function test_digitalocean_client_test_connection_if_not_configured_prope $this->assertSame(get_string('settings:notconfigured', 'tool_objectfs'), $testresults->details); } + /** + * Provides values to azure_client_get_token_expiry test + * @return array + */ + public static function azure_client_get_token_expiry_provider(): array { + return [ + 'good token' => [ + 'sastoken' => 'sp=racwl&st=2024-09-13T01:11:06Z&se=2024-12-30T09:11:06Z&spr=https&sv=2022-11-02&sr=c&sig=abcd', + 'expectedtime' => 1735549866, + ], + 'malformed se' => [ + 'sastoken' => 'sp=racwl&st=2024-09-13T01:11:06Z&se=-12-30T09:11:06Z&spr=https&sv=2022-11-02&sr=c&sig=abcd', + 'expectedtime' => 0, + ], + 'missing se' => [ + 'sastoken' => 'sp=racwl&st=2024-09-13T01:11:06Z&spr=https&sv=2022-11-02&sr=c&sig=abcd', + 'expectedtime' => 0, + ], + 'no token' => [ + 'sastoken' => '', + 'expectedtime' => -1, + ], + ]; + } + + /** + * Tests azure client correctly extracts token expiry time + * @param string $sastoken + * @param int $expectedtime + * @dataProvider azure_client_get_token_expiry_provider + */ + public function test_azure_client_get_token_expiry(string $sastoken, int $expectedtime) { + $config = (object) [ + 'azure_container' => 'test', + 'azure_accountname' => 'test', + 'azure_sastoken' => $sastoken, + ]; + $azureclient = new azureclient($config); + $this->assertEquals($expectedtime, $azureclient->get_token_expiry_time()); + } } diff --git a/tests/token_expiry_test.php b/tests/token_expiry_test.php new file mode 100644 index 00000000..a658bd0f --- /dev/null +++ b/tests/token_expiry_test.php @@ -0,0 +1,77 @@ +. + +namespace tool_objectfs; + +use core\check\result; +use tool_objectfs\check\token_expiry; +use tool_objectfs\local\manager; +use tool_objectfs\tests\testcase; + +/** + * Token expiry check test. + * + * @covers \tool_objectfs\check\token_expiry + * @package tool_objectfs + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class token_expiry_test extends testcase { + /** + * Provides to test_get_result + * @return array + */ + public static function get_result_provider(): array { + return [ + 'ok' => [ + 'expiry' => time() + 10 * DAYSECS, + 'warnperiod' => 5 * DAYSECS, + 'expectedresult' => result::OK, + ], + 'warning' => [ + 'expiry' => time() + DAYSECS, + 'warnperiod' => 5 * DAYSECS, + 'expectedresult' => result::WARNING, + ], + 'expired' => [ + 'expiry' => time() - DAYSECS, + 'warnperiod' => 5 * DAYSECS, + 'expectedresult' => result::CRITICAL, + ], + ]; + } + + /** + * Tests getting check result + * @param int $expirytime time to use as the tokens expiry + * @param int $warnperiod period to set for warning about token expiry + * @param string $expectedresult one of the result:: constants that is expected to be returned. + * @dataProvider get_result_provider + */ + public function test_get_result(int $expirytime, int $warnperiod, string $expectedresult) { + global $CFG; + $config = manager::get_objectfs_config(); + $config->filesystem = '\\tool_objectfs\\tests\\test_file_system'; + manager::set_objectfs_config($config); + + $CFG->objectfs_phpunit_token_expiry_time = $expirytime; + set_config('tokenexpirywarnperiod', $warnperiod, 'tool_objectfs'); + + $check = new token_expiry(); + $result = $check->get_result(); + $this->assertEquals($expectedresult, $result->get_status()); + } +} diff --git a/version.php b/version.php index bea5f7e8..25c5d28d 100644 --- a/version.php +++ b/version.php @@ -25,8 +25,8 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2023051701; // The current plugin version (Date: YYYYMMDDXX). -$plugin->release = 2023051701; // Same as version. +$plugin->version = 2024091700; // The current plugin version (Date: YYYYMMDDXX). +$plugin->release = 2024091700; // Same as version. $plugin->requires = 2023042400; // Requires 4.2. $plugin->component = "tool_objectfs"; $plugin->maturity = MATURITY_STABLE; From c0f6c05d230b61267ec7c21b0e7368ac1e41b8c4 Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Tue, 17 Sep 2024 14:35:44 +1000 Subject: [PATCH 12/28] chore: code standards fixup --- .../candidates/manipulator_candidates.php | 9 ++++----- classes/local/object_manipulator/object_manipulator.php | 9 ++++----- classes/local/store/object_client.php | 6 ++---- classes/privacy/provider.php | 1 + classes/task/objectfs_task.php | 5 ++--- classes/tests/test_azure_integration_client.php | 7 +++++-- classes/tests/test_client.php | 7 +++++-- classes/tests/test_digitalocean_integration_client.php | 7 +++++-- classes/tests/test_s3_integration_client.php | 7 +++++-- classes/tests/test_swift_integration_client.php | 7 +++++-- classes/tests/testcase.php | 7 +++++-- tests/local/object_manipulator/checker_test.php | 4 +++- tests/local/object_manipulator/deleter_test.php | 4 +++- tests/local/object_manipulator/orphaner_test.php | 4 +++- tests/local/object_manipulator/puller_test.php | 4 +++- tests/local/object_manipulator/pusher_test.php | 4 +++- tests/local/object_manipulator/recoverer_test.php | 4 +++- tests/local/tasks_test.php | 4 +++- tests/object_file_system_test.php | 4 +++- 19 files changed, 67 insertions(+), 37 deletions(-) diff --git a/classes/local/object_manipulator/candidates/manipulator_candidates.php b/classes/local/object_manipulator/candidates/manipulator_candidates.php index eac7ca90..6cb68bc9 100644 --- a/classes/local/object_manipulator/candidates/manipulator_candidates.php +++ b/classes/local/object_manipulator/candidates/manipulator_candidates.php @@ -14,6 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Moodle. If not, see . +namespace tool_objectfs\local\object_manipulator\candidates; + +use dml_exception; + /** * Interface manipulator_candidates * @package tool_objectfs @@ -21,11 +25,6 @@ * @copyright Catalyst IT * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ - -namespace tool_objectfs\local\object_manipulator\candidates; - -use dml_exception; - interface manipulator_candidates { /** diff --git a/classes/local/object_manipulator/object_manipulator.php b/classes/local/object_manipulator/object_manipulator.php index 33d9d88a..8549106e 100644 --- a/classes/local/object_manipulator/object_manipulator.php +++ b/classes/local/object_manipulator/object_manipulator.php @@ -14,6 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Moodle. If not, see . +namespace tool_objectfs\local\object_manipulator; + +use stdClass; + /** * Object manipulator interface class. * @@ -22,11 +26,6 @@ * @copyright Catalyst IT * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ - -namespace tool_objectfs\local\object_manipulator; - -use stdClass; - interface object_manipulator { diff --git a/classes/local/store/object_client.php b/classes/local/store/object_client.php index cd8cbc7f..80e3f6eb 100644 --- a/classes/local/store/object_client.php +++ b/classes/local/store/object_client.php @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Moodle. If not, see . +namespace tool_objectfs\local\store; + /** * Objectfs client interface. * @@ -22,11 +24,7 @@ * @copyright Catalyst IT * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ - -namespace tool_objectfs\local\store; - interface object_client { - /** * construct * @param \stdClass $config diff --git a/classes/privacy/provider.php b/classes/privacy/provider.php index 2e6d16c6..ecfd06c3 100644 --- a/classes/privacy/provider.php +++ b/classes/privacy/provider.php @@ -13,6 +13,7 @@ // // You should have received a copy of the GNU General Public License // along with Moodle. If not, see . + /** * Privacy provider. * diff --git a/classes/task/objectfs_task.php b/classes/task/objectfs_task.php index 33b8fb4a..86d39b2f 100644 --- a/classes/task/objectfs_task.php +++ b/classes/task/objectfs_task.php @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Moodle. If not, see . +namespace tool_objectfs\task; + /** * Base abstract class for objectfs tasks. * @@ -22,9 +24,6 @@ * @copyright Catalyst IT * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ - -namespace tool_objectfs\task; - interface objectfs_task { /** diff --git a/classes/tests/test_azure_integration_client.php b/classes/tests/test_azure_integration_client.php index cec69065..74965b27 100644 --- a/classes/tests/test_azure_integration_client.php +++ b/classes/tests/test_azure_integration_client.php @@ -19,8 +19,11 @@ use tool_objectfs\local\store\azure\client; /** - * [Description test_azure_integration_client] - * @package tool_objectfs + * Client used for integration testing azure client + * + * @package tool_objectfs + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class test_azure_integration_client extends client { diff --git a/classes/tests/test_client.php b/classes/tests/test_client.php index dbed5711..bda43c5e 100644 --- a/classes/tests/test_client.php +++ b/classes/tests/test_client.php @@ -19,8 +19,11 @@ use tool_objectfs\local\store\object_client_base; /** - * [Description test_client] - * @package tool_objectfs + * Test client for PHP unit tests + * + * @package tool_objectfs + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class test_client extends object_client_base { /** diff --git a/classes/tests/test_digitalocean_integration_client.php b/classes/tests/test_digitalocean_integration_client.php index 2a34c28c..5bf08cad 100644 --- a/classes/tests/test_digitalocean_integration_client.php +++ b/classes/tests/test_digitalocean_integration_client.php @@ -19,8 +19,11 @@ use tool_objectfs\local\store\digitalocean\client; /** - * [Description test_digitalocean_integration_client] - * @package tool_objectfs + * Client used for integration testing digitalocean client + * + * @package tool_objectfs + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class test_digitalocean_integration_client extends client { diff --git a/classes/tests/test_s3_integration_client.php b/classes/tests/test_s3_integration_client.php index 233f0b7e..06943973 100644 --- a/classes/tests/test_s3_integration_client.php +++ b/classes/tests/test_s3_integration_client.php @@ -19,8 +19,11 @@ use tool_objectfs\local\store\s3\client; /** - * [Description test_s3_integration_client] - * @package tool_objectfs + * Client used for integration testing aws client + * + * @package tool_objectfs + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class test_s3_integration_client extends client { diff --git a/classes/tests/test_swift_integration_client.php b/classes/tests/test_swift_integration_client.php index 4e1ede63..3013a299 100644 --- a/classes/tests/test_swift_integration_client.php +++ b/classes/tests/test_swift_integration_client.php @@ -19,8 +19,11 @@ use tool_objectfs\local\store\swift\client; /** - * [Description test_swift_integration_client] - * @package tool_objectfs + * Client used for integration testing swift client + * + * @package tool_objectfs + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class test_swift_integration_client extends client { diff --git a/classes/tests/testcase.php b/classes/tests/testcase.php index 6d85a3b5..f3efc8cf 100644 --- a/classes/tests/testcase.php +++ b/classes/tests/testcase.php @@ -26,8 +26,11 @@ use tool_objectfs\local\store\signed_url; /** - * [Description testcase] - * @package tool_objectfs + * Testcase with useful / shared methods for common objectfs tests. + * + * @package tool_objectfs + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ abstract class testcase extends \advanced_testcase { diff --git a/tests/local/object_manipulator/checker_test.php b/tests/local/object_manipulator/checker_test.php index 7ac8f98d..59699d8f 100644 --- a/tests/local/object_manipulator/checker_test.php +++ b/tests/local/object_manipulator/checker_test.php @@ -22,7 +22,9 @@ * Tests for object checker. * * @covers \tool_objectfs\local\object_manipulator\checker - * @package tool_objectfs + * @package tool_objectfs + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class checker_test extends \tool_objectfs\tests\testcase { diff --git a/tests/local/object_manipulator/deleter_test.php b/tests/local/object_manipulator/deleter_test.php index 56ddfa5b..a8b3f7a1 100644 --- a/tests/local/object_manipulator/deleter_test.php +++ b/tests/local/object_manipulator/deleter_test.php @@ -22,7 +22,9 @@ * Tests for object deleter. * * @covers \tool_objectfs\local\object_manipulator\deleter - * @package tool_objectfs + * @package tool_objectfs + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class deleter_test extends \tool_objectfs\tests\testcase { diff --git a/tests/local/object_manipulator/orphaner_test.php b/tests/local/object_manipulator/orphaner_test.php index 1498e0ff..9485b175 100644 --- a/tests/local/object_manipulator/orphaner_test.php +++ b/tests/local/object_manipulator/orphaner_test.php @@ -23,7 +23,9 @@ * Tests for object orphaner. * * @covers \tool_objectfs\local\object_manipulator\orphaner - * @package tool_objectfs + * @package tool_objectfs + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class orphaner_test extends \tool_objectfs\tests\testcase { diff --git a/tests/local/object_manipulator/puller_test.php b/tests/local/object_manipulator/puller_test.php index d54171cb..e00a81b9 100644 --- a/tests/local/object_manipulator/puller_test.php +++ b/tests/local/object_manipulator/puller_test.php @@ -22,7 +22,9 @@ * Tests for object puller. * * @covers \tool_objectfs\local\object_manipulator\puller - * @package tool_objectfs + * @package tool_objectfs + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class puller_test extends \tool_objectfs\tests\testcase { diff --git a/tests/local/object_manipulator/pusher_test.php b/tests/local/object_manipulator/pusher_test.php index e902f0be..37e832ba 100644 --- a/tests/local/object_manipulator/pusher_test.php +++ b/tests/local/object_manipulator/pusher_test.php @@ -23,7 +23,9 @@ * Tests for object pusher. * * @covers \tool_objectfs\local\object_manipulator\pusher - * @package tool_objectfs + * @package tool_objectfs + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class pusher_test extends \tool_objectfs\tests\testcase { diff --git a/tests/local/object_manipulator/recoverer_test.php b/tests/local/object_manipulator/recoverer_test.php index 1b8f7927..163677a4 100644 --- a/tests/local/object_manipulator/recoverer_test.php +++ b/tests/local/object_manipulator/recoverer_test.php @@ -23,7 +23,9 @@ * Tests for object recoverer. * * @covers \tool_objectfs\local\object_manipulator\recoverer - * @package tool_objectfs + * @package tool_objectfs + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class recoverer_test extends \tool_objectfs\tests\testcase { diff --git a/tests/local/tasks_test.php b/tests/local/tasks_test.php index afdc53b2..10f884fa 100644 --- a/tests/local/tasks_test.php +++ b/tests/local/tasks_test.php @@ -20,7 +20,9 @@ * End to end tests for tasks. Make sure all the plumbing is ok. * * @covers \tool_objectfs\local\manager - * @package tool_objectfs + * @package tool_objectfs + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class tasks_test extends \tool_objectfs\tests\testcase { diff --git a/tests/object_file_system_test.php b/tests/object_file_system_test.php index 0ccd2a5a..3621c064 100644 --- a/tests/object_file_system_test.php +++ b/tests/object_file_system_test.php @@ -24,7 +24,9 @@ * Test basic operations of object file system. * * @covers \tool_objectfs\local\store\object_file_system - * @package tool_objectfs + * @package tool_objectfs + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class object_file_system_test extends tests\testcase { From cf91aa2cd93afa0c31236cb958f7c405f85c501c Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Tue, 17 Sep 2024 15:22:40 +1000 Subject: [PATCH 13/28] bugfix: load azure client in unit tests --- classes/tests/test_azure_integration_client.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/classes/tests/test_azure_integration_client.php b/classes/tests/test_azure_integration_client.php index 74965b27..c871e1f4 100644 --- a/classes/tests/test_azure_integration_client.php +++ b/classes/tests/test_azure_integration_client.php @@ -38,7 +38,10 @@ class test_azure_integration_client extends client { * @return void */ public function __construct($config) { - parent::__construct($config); + // Set config directly. Calling __construct will do nothing + // since unit tests do not have the azure sdk installed. + $this->config = $config; + $time = microtime(); $this->runidentifier = md5($time); } From ad92e2aba838eab55b3e4ee226ea7f84a0ce0649 Mon Sep 17 00:00:00 2001 From: Peter Sistrom Date: Thu, 19 Sep 2024 11:13:23 +1000 Subject: [PATCH 14/28] Issue #634: Return external accessible false is file empty or directory --- classes/local/store/object_file_system.php | 2 +- tests/object_file_system_test.php | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/classes/local/store/object_file_system.php b/classes/local/store/object_file_system.php index de30f1d2..38e12a2c 100644 --- a/classes/local/store/object_file_system.php +++ b/classes/local/store/object_file_system.php @@ -249,7 +249,7 @@ public function is_file_readable_externally_by_hash($contenthash) { if ($contenthash === sha1('')) { // Files with empty size are either directories or empty. // We handle these virtually. - return true; + return false; } $path = $this->get_external_path_from_hash($contenthash, false); diff --git a/tests/object_file_system_test.php b/tests/object_file_system_test.php index 3621c064..6843bd6b 100644 --- a/tests/object_file_system_test.php +++ b/tests/object_file_system_test.php @@ -86,6 +86,19 @@ public function test_get_remote_path_from_storedfile_returns_external_path_if_du $this->assertEquals($expectedpath, $actualpath); } + public function test_get_remote_path_from_empty_storedfile_returns_internal_path_if_duplicated_and_preferexternal() { + set_config('preferexternal', true, 'tool_objectfs'); + $this->reset_file_system(); // Needed to load new config. + $file = $this->create_duplicated_file(''); + $expectedpath = $this->get_local_path_from_storedfile($file); + + $reflection = new \ReflectionMethod(object_file_system::class, 'get_remote_path_from_storedfile'); + $reflection->setAccessible(true); + $actualpath = $reflection->invokeArgs($this->filesystem, [$file]); + + $this->assertEquals($expectedpath, $actualpath); + } + public function test_get_local_path_from_hash_will_fetch_remote_if_fetchifnotfound() { $file = $this->create_remote_file(); $filehash = $file->get_contenthash(); From 3b0b1cc275be39f87379f73add6edb20c548d549 Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Thu, 24 Oct 2024 10:22:13 +1000 Subject: [PATCH 15/28] bugfix: fix token expiry check in admin settings --- classes/check/token_expiry.php | 10 ++++++++++ classes/local/store/azure/client.php | 11 ----------- lang/en/tool_objectfs.php | 4 +++- settings.php | 14 ++++++++++++++ version.php | 4 ++-- 5 files changed, 29 insertions(+), 14 deletions(-) diff --git a/classes/check/token_expiry.php b/classes/check/token_expiry.php index ad6e8d6d..2eda943c 100644 --- a/classes/check/token_expiry.php +++ b/classes/check/token_expiry.php @@ -28,6 +28,16 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class token_expiry extends check { + /** + * Link to ObjectFS settings page. + * + * @return \action_link|null + */ + public function get_action_link(): ?\action_link { + $url = new \moodle_url('/admin/category.php', ['category' => 'tool_objectfs']); + return new \action_link($url, get_string('pluginname', 'tool_objectfs')); + } + /** * Checks the token expiry time against thresholds * @return result diff --git a/classes/local/store/azure/client.php b/classes/local/store/azure/client.php index ed755e0f..047b07d7 100644 --- a/classes/local/store/azure/client.php +++ b/classes/local/store/azure/client.php @@ -25,10 +25,8 @@ namespace tool_objectfs\local\store\azure; -use admin_setting_description; use SimpleXMLElement; use stdClass; -use tool_objectfs\check\token_expiry; use tool_objectfs\local\store\azure\stream_wrapper; use tool_objectfs\local\store\object_client_base; @@ -362,15 +360,6 @@ public function define_client_section($settings, $config) { new \lang_string('settings:azure:sastoken', 'tool_objectfs'), new \lang_string('settings:azure:sastoken_help', 'tool_objectfs'), '')); - // Admin_setting_check only exists in 4.5+, in lower versions fallback to a basic description. - if (class_exists('admin_setting_check')) { - $settings->add(new admin_setting_check('tool_objectfs/check_tokenexpiry', new token_expiry(), true)); - } else { - $summary = (new token_expiry())->get_result()->get_summary(); - $settings->add(new admin_setting_description('tool_objectfs/tokenexpirycheckresult', - get_string('checktoken_expiry', 'tool_objectfs'), $summary)); - } - return $settings; } diff --git a/lang/en/tool_objectfs.php b/lang/en/tool_objectfs.php index 74b383a5..7a341a91 100644 --- a/lang/en/tool_objectfs.php +++ b/lang/en/tool_objectfs.php @@ -262,6 +262,8 @@ $string['settings:testingheader'] = 'Test Settings'; $string['settings:testingdescr'] = 'This setting is mainly for testing purposes and introduces overhead to check the location.'; +$string['settings:checksheader'] = 'Checks'; + $string['settings:error:numeric'] = 'Please enter a number which is greater than or equal 0.'; $string['settings:notconfigured'] = 'Missing configuration.'; $string['total_deleted_dirs'] = 'Total number of deleted directories: '; @@ -273,5 +275,5 @@ $string['checktoken_expiry'] = 'Token expiry'; $string['check:tokenexpiry:expiresin'] = 'Token expires in {$a->dayssince} days on {$a->time}'; $string['check:tokenexpiry:expired'] = 'Token expired for {$a->dayssince} days. Expired on {$a->time}'; -$string['check:tokenexpiry:na'] = 'Token expired not implemented for filesystem, or no token is set'; +$string['check:tokenexpiry:na'] = 'Token expiry check not implemented for filesystem, or no token is set'; $string['settings:tokenexpirywarnperiod'] = 'Token expiry warn period'; diff --git a/settings.php b/settings.php index aef3c571..5ed74fef 100644 --- a/settings.php +++ b/settings.php @@ -23,6 +23,8 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ +use tool_objectfs\check\token_expiry; + defined('MOODLE_INTERNAL') || die(); require_once(__DIR__ . '/classes/local/manager.php'); @@ -248,6 +250,18 @@ } } + $settings->add(new admin_setting_heading('tool_objectfs/checks', + new lang_string('settings:checksheader', 'tool_objectfs'), '')); + + // Admin_setting_check only exists in 4.5+, in lower versions fallback to a basic description. + if (class_exists('admin_setting_check')) { + $settings->add(new admin_setting_check('tool_objectfs/check_tokenexpiry', new token_expiry(), true)); + } else { + $summary = (new token_expiry())->get_result()->get_summary(); + $settings->add(new admin_setting_description('tool_objectfs/tokenexpirycheckresult', + get_string('checktoken_expiry', 'tool_objectfs'), $summary)); + } + $settings->add(new admin_setting_heading('tool_objectfs/testsettings', new lang_string('settings:testingheader', 'tool_objectfs'), '')); diff --git a/version.php b/version.php index 25c5d28d..09084a53 100644 --- a/version.php +++ b/version.php @@ -25,8 +25,8 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2024091700; // The current plugin version (Date: YYYYMMDDXX). -$plugin->release = 2024091700; // Same as version. +$plugin->version = 2024102400; // The current plugin version (Date: YYYYMMDDXX). +$plugin->release = 2024102400; // Same as version. $plugin->requires = 2023042400; // Requires 4.2. $plugin->component = "tool_objectfs"; $plugin->maturity = MATURITY_STABLE; From 73a41ffc3613bf68450f77e0891815fd38623094 Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Tue, 15 Oct 2024 11:48:01 +1000 Subject: [PATCH 16/28] bugfix: test delete according to objectfs config --- classes/local/store/digitalocean/client.php | 1 - classes/local/store/object_client.php | 6 ++++++ classes/local/store/object_client_base.php | 15 ++++++++++----- classes/local/store/s3/client.php | 2 +- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/classes/local/store/digitalocean/client.php b/classes/local/store/digitalocean/client.php index 226636d6..f7946d34 100644 --- a/classes/local/store/digitalocean/client.php +++ b/classes/local/store/digitalocean/client.php @@ -40,7 +40,6 @@ class client extends s3_client { public function __construct($config) { global $CFG; $this->autoloader = $CFG->dirroot . '/local/aws/sdk/aws-autoloader.php'; - $this->testdelete = false; if ($this->get_availability() && !empty($config)) { require_once($this->autoloader); diff --git a/classes/local/store/object_client.php b/classes/local/store/object_client.php index 80e3f6eb..cec71d77 100644 --- a/classes/local/store/object_client.php +++ b/classes/local/store/object_client.php @@ -141,6 +141,12 @@ public function test_range_request($filesystem); * @return int unix timestamp the token set expires at */ public function get_token_expiry_time(): int; + + /** + * If should test deletion as part of testing connection. + * @return bool + */ + public function should_test_delete(): bool; } diff --git a/classes/local/store/object_client_base.php b/classes/local/store/object_client_base.php index fa1b7e8f..0dabbc17 100644 --- a/classes/local/store/object_client_base.php +++ b/classes/local/store/object_client_base.php @@ -38,10 +38,6 @@ abstract class object_client_base implements object_client { * @var mixed */ protected $expirationtime; - /** - * @var bool - */ - protected $testdelete = true; /** * @var int */ @@ -57,6 +53,15 @@ abstract class object_client_base implements object_client { /** @var object $config Client config. */ protected $config; + /** + * If deletion should be tested. + * Only tested when the deleteexternal setting is not set to 'no' + * @return bool + */ + public function should_test_delete(): bool { + return !empty($this->config) && $this->config->deleteexternal != TOOL_OBJECTFS_DELETE_EXTERNAL_NO; + } + /** * construct * @param \stdClass $config @@ -120,7 +125,7 @@ public function define_client_check() { if ($connection->success) { $output .= $OUTPUT->notification(get_string('settings:connectionsuccess', 'tool_objectfs'), 'notifysuccess'); // Check permissions if we can connect. - $permissions = $this->test_permissions($this->testdelete); + $permissions = $this->test_permissions($this->should_test_delete()); if ($permissions->success) { $output .= $OUTPUT->notification(key($permissions->messages), 'notifysuccess'); } else { diff --git a/classes/local/store/s3/client.php b/classes/local/store/s3/client.php index a6e2598e..23c495af 100644 --- a/classes/local/store/s3/client.php +++ b/classes/local/store/s3/client.php @@ -767,7 +767,7 @@ public function define_client_check_sdk($config) { if ($connection->success) { $output .= $OUTPUT->notification(get_string('settings:aws:sdkcredsok', 'tool_objectfs'), 'notifysuccess'); // Check permissions if we can connect. - $permissions = $this->test_permissions($this->testdelete); + $permissions = $this->test_permissions($this->should_test_delete()); if ($permissions->success) { $output .= $OUTPUT->notification(key($permissions->messages), 'notifysuccess'); } else { From 5cb99d20b4ed27ed355e7719ea2918ef6b1d14ef Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Thu, 31 Oct 2024 13:42:02 +1000 Subject: [PATCH 17/28] bugfix: fix test assumption --- tests/task/populate_objects_filesize_test.php | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/task/populate_objects_filesize_test.php b/tests/task/populate_objects_filesize_test.php index 9ff08ee0..fdb9d62d 100644 --- a/tests/task/populate_objects_filesize_test.php +++ b/tests/task/populate_objects_filesize_test.php @@ -179,24 +179,27 @@ public function test_that_non_null_values_are_not_updated() { */ public function test_orphaned_objects_are_not_updated() { global $DB; - $file1 = $this->create_local_file("Test 1"); - $this->create_local_file("Test 2"); - $this->create_local_file("Test 3"); - $this->create_local_file("Test 4"); - $this->create_local_file("This is a looong name"); + $filehashes = [ + $this->create_local_file("Test 1")->get_contenthash(), + $this->create_local_file("Test 2")->get_contenthash(), + $this->create_local_file("Test 3")->get_contenthash(), + $this->create_local_file("Test 4")->get_contenthash(), + $this->create_local_file("This is a looong name")->get_contenthash(), + ]; // Set all objects to have a filesize of null. $DB->set_field('tool_objectfs_objects', 'filesize', null); // Set first object to be orphaned. - $DB->set_field('tool_objectfs_objects', 'location', -2, ['contenthash' => $file1->get_contenthash()]); + $DB->set_field('tool_objectfs_objects', 'location', -2, ['contenthash' => $filehashes[0]]); // Call ad-hoc task to populate filesizes. $task = new \tool_objectfs\task\populate_objects_filesize(); $task->execute(); // Get all objects. - $objects = $DB->get_records('tool_objectfs_objects'); + [$insql, $params] = $DB->get_in_or_equal($filehashes); + $objects = $DB->get_records_select('tool_objectfs_objects', 'contenthash ' . $insql, $params); $updatedobjects = array_filter($objects, function($object) { return isset($object->filesize); }); From 47c8fdca51d699105784d7eef90fbf0855f115b2 Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Tue, 15 Oct 2024 11:48:17 +1000 Subject: [PATCH 18/28] feat: add new azure sdk, deprecate old sdk --- MIGRATION.md | 17 + README.md | 72 +++-- TESTING.md | 10 +- classes/azure_blob_storage_file_system.php | 33 ++ classes/local/manager.php | 7 +- classes/local/store/azure/client.php | 1 + classes/local/store/azure/file_system.php | 1 + classes/local/store/azure/stream_wrapper.php | 1 + .../local/store/azure_blob_storage/client.php | 296 ++++++++++++++++++ .../store/azure_blob_storage/file_system.php | 39 +++ ..._azure_blob_storage_integration_client.php | 60 ++++ .../tests/test_azure_integration_client.php | 1 + classes/tests/test_file_system.php | 7 + tests/object_file_system_test.php | 7 + 14 files changed, 521 insertions(+), 31 deletions(-) create mode 100644 MIGRATION.md create mode 100644 classes/azure_blob_storage_file_system.php create mode 100644 classes/local/store/azure_blob_storage/client.php create mode 100644 classes/local/store/azure_blob_storage/file_system.php create mode 100644 classes/tests/test_azure_blob_storage_integration_client.php diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 00000000..6e2e08e5 --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,17 @@ +# Migration guides + +## Migrating from local_azure_storage to local_azureblobstorage + +Since March 2024, Microsoft officially discontinued support for the PHP SDK for Azure storage. This means the `local_azure_storage` plugin which is a wrapper of the SDK will no longer be updated, and is already out of date with newer php versions. + +The plugin `local_azureblobstorage` was created to replace this, with a simpler and cleaner API for interacting with the Azure blob storage service via REST APIs. Objectfs has been updated with a new client handler class to enable you to cut over to the new storage system as easily as possible. + +This new library is only supported in higher PHP versions. + +### Steps +1. If you are on Moodle 4.2, ensure you have updated the previous `local_azure_storage` to the `MOODLE_42_STABLE` branch. This fixes some fatal errors caused by Guzzle namespace conflicts. +2. Install `local_azureblobstorage` https://github.com/catalyst/moodle-local_azureblobstorage +3. In the objectfs settings, change the `filesystem` config variable to `\tool_objectfs\azure_blob_storage_file_system` and save. ObjectFS will now be using the new API to communicate with Azure. You do not need to enter new credentials, the credentials are shared with the old client. +4. Test and ensure the site works as expected. +5. If you encounter any issues and wish to revert back, simply change the `filesystem` configuration back to the old client. This will immediately begin to use the old libraries again. +6. Once you are happy, simply uninstall the `local_azure_storage` plugin. The migration is now complete. \ No newline at end of file diff --git a/README.md b/README.md index 3f52804f..348e68e4 100644 --- a/README.md +++ b/README.md @@ -5,31 +5,45 @@ # moodle-tool_objectfs A remote object storage file system for Moodle. Intended to provide a plug-in that can be installed and configured to work with any supported remote object storage solution. -* [Use cases](#use-cases) - * [Offloading large and old files to save money](#offloading-large-and-old-files-to-save-money) - * [Sharing files across moodles to save disk](#sharing-files-across-moodles-to-save-disk) - * [Sharing files across environments to save time](#sharing-files-across-environments-to-save-time) - * [Sharing files with data washed environments](#sharing-files-with-data-washed-environments) -* [Installation](#installation) -* [Compatible object stores](#compatible-object-stores) - * [Amazon S3](#amazon-s3) - * [Minio.io S3](#minio-s3) - * [Google gcs](#google-gcs) - * [Azure Blob Storage](#azure-blob-storage) - * [DigitalOcean Spaces](#digitalocean-spaces) - * [Openstack Object Storage](#openstack-object-storage) -* [Moodle configuration](#moodle-configuration) - * [General Settings](#general-settings) - * [File Transfer settings](#file-transfer-settings) - * [Pre-Signed URLs Settings](#pre-signed-urls-settings) - * [Amazon S3 settings](#amazon-s3-settings) - * [Minio.io S3 settings](#minio-s3-settings) - * [Azure Blob Storage settings](#azure-blob-storage-settings) - * [DigitalOcean Spaces settings](#digitalocean-spaces-settings) -* [Integration testing](#integration-testing) -* [Applying core patches](#applying-core-patches) -* [Crafted by Catalyst IT](#crafted-by-catalyst-it) -* [Contributing and support](#contributing-and-support) +- [moodle-tool\_objectfs](#moodle-tool_objectfs) + - [Use cases](#use-cases) + - [Offloading large and old files to save money](#offloading-large-and-old-files-to-save-money) + - [Sharing files across moodles to save disk](#sharing-files-across-moodles-to-save-disk) + - [Sharing files across environments to save time](#sharing-files-across-environments-to-save-time) + - [Sharing files with data washed environments](#sharing-files-with-data-washed-environments) + - [GDPR](#gdpr) + - [Branches](#branches) + - [Installation](#installation) + - [Compatible object stores](#compatible-object-stores) + - [Amazon S3](#amazon-s3) + - [Minio S3](#minio-s3) + - [Google GCS](#google-gcs) + - [Azure Blob Storage](#azure-blob-storage) + - [DigitalOcean Spaces](#digitalocean-spaces) + - [Openstack Object Storage](#openstack-object-storage) + - [Moodle configuration](#moodle-configuration) + - [General Settings](#general-settings) + - [File Transfer settings](#file-transfer-settings) + - [File System settings](#file-system-settings) + - [Pre-Signed URLs Settings](#pre-signed-urls-settings) + - [Amazon S3 settings](#amazon-s3-settings) + - [Minio S3 settings](#minio-s3-settings) + - [Azure Blob Storage settings](#azure-blob-storage-settings) + - [DigitalOcean Spaces settings](#digitalocean-spaces-settings) + - [Openstack Object Storage settings](#openstack-object-storage-settings) + - [Integration testing](#integration-testing) + - [Applying core patches](#applying-core-patches) + - [Moodle 3.9:](#moodle-39) + - [Moodle 3.8:](#moodle-38) + - [Moodle 3.4 - 3.7:](#moodle-34---37) + - [Moodle 3.3 and Totara 12:](#moodle-33-and-totara-12) + - [Moodle 3.2 and Totara 11:](#moodle-32-and-totara-11) + - [Moodle 2.9 - 3.1 and Totara 2.9, 9 - 10:](#moodle-29---31-and-totara-29-9---10) + - [Moodle 2.7 - 2.8 and Totara 2.7 - 2.8:](#moodle-27---28-and-totara-27---28) + - [PHPUnit test compatibility](#phpunit-test-compatibility) + - [Contributing and support](#contributing-and-support) + - [Warm thanks](#warm-thanks) + - [Crafted by Catalyst IT](#crafted-by-catalyst-it) ## Use cases There are a number of different ways you can use this plug in. See [Recommended use case settings](#recommended-use-case-settings) for recommended settings for each one. @@ -75,7 +89,7 @@ This plugin is GDPR complient if you enable the deletion of remote objects. 3. Clone this repository into admin/tool/objectfs 4. Install one of the required SDK libraries for the storage file system that you will be using 1. Clone [moodle-local_aws](https://github.com/catalyst/moodle-local_aws) into local/aws for S3 or DigitalOcean Spaces or Google Cloud, or - 2. Clone [moodle-local_azure_storage](https://github.com/catalyst/moodle-local_azure_storage) into local/azure_storage for Azure Blob Storage, or + 2. Clone [moodle-local_azureblobstorage](https://github.com/catalyst/moodle-local_azureblobstorage) into local/azureblobstorage for Azure Blob Storage, or 3. Clone [moodle-local_openstack](https://github.com/matt-catalyst/moodle-local_openstack.git) into local/openstack for openstack(swift) storage 5. Install the plugins through the moodle GUI. 6. Configure the plugin. See [Moodle configuration](#moodle-configuration) @@ -88,7 +102,7 @@ $CFG->alternative_file_system_class = '\tool_objectfs\s3_file_system'; * Azure Blob Storage ```php -$CFG->alternative_file_system_class = '\tool_objectfs\azure_file_system'; +$CFG->alternative_file_system_class = '\tool_objectfs\azure_blob_storage_file_system'; ``` * DigitalOcean Spaces @@ -155,6 +169,10 @@ Setup for Minio.io bucket can be found on there website [here](https://min.io) ### Azure Blob Storage +*Migration from previous API (Moodle 4.2+)* + +[Migration guide from local_azure_storage to local_azureblobstorage](MIGRATION#migrating-from-localazurestorage-to-localazureblobstorage) + *Azure Storage container guide with the CLI* It is possible to install the Azure CLI locally to administer the storage account. [The Azure CLI can be obtained here.](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest) diff --git a/TESTING.md b/TESTING.md index 2c20097c..2b59be30 100644 --- a/TESTING.md +++ b/TESTING.md @@ -15,7 +15,7 @@ $CFG->phpunit_objectfs_s3_integration_test_credentials = array( 's3_region' => 'Your region', ); ``` -* Azure: +* Azure (deprecated API): ```php $CFG->phpunit_objectfs_azure_integration_test_credentials = array( 'azure_accountname' => 'Your account name', @@ -23,6 +23,14 @@ $CFG->phpunit_objectfs_azure_integration_test_credentials = array( 'azure_sastoken' => 'Your sas token', ); ``` +* Azure Blob Storage: +```php +$CFG->phpunit_objectfs_azure_blob_storage_integration_test_credentials = [ + 'azure_accountname' => 'Your account name', + 'azure_container' => 'Your container', + 'azure_sastoken' => 'Your sas token', +]; +``` * Swift: ```php $CFG->phpunit_objectfs_swift_integration_test_credentials = array( diff --git a/classes/azure_blob_storage_file_system.php b/classes/azure_blob_storage_file_system.php new file mode 100644 index 00000000..b0c3d751 --- /dev/null +++ b/classes/azure_blob_storage_file_system.php @@ -0,0 +1,33 @@ +. + +namespace tool_objectfs; + +use tool_objectfs\local\store\azure_blob_storage\file_system; + +/** + * File system for Azure Blob Storage. + * + * This file tells objectfs that this storage system is available for use. + * E.g. via $CFG->alternative_file_system_class + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class azure_blob_storage_file_system extends file_system { +} diff --git a/classes/local/manager.php b/classes/local/manager.php index acd2b30e..ae763fda 100644 --- a/classes/local/manager.php +++ b/classes/local/manager.php @@ -306,17 +306,18 @@ public static function check_file_storage_filesystem() { public static function get_available_fs_list() { $result[''] = get_string('pleaseselect', OBJECTFS_PLUGIN_NAME); - $filesystems['\tool_objectfs\azure_file_system'] = '\tool_objectfs\azure_file_system'; + $filesystems['\tool_objectfs\azure_file_system'] = '\tool_objectfs\azure_file_system [DEPRECATED]'; $filesystems['\tool_objectfs\digitalocean_file_system'] = '\tool_objectfs\digitalocean_file_system'; $filesystems['\tool_objectfs\s3_file_system'] = '\tool_objectfs\s3_file_system'; $filesystems['\tool_objectfs\swift_file_system'] = '\tool_objectfs\swift_file_system'; + $filesystems['\tool_objectfs\azure_blob_storage_file_system'] = '\tool_objectfs\azure_blob_storage_file_system'; - foreach ($filesystems as $filesystem) { + foreach ($filesystems as $filesystem => $name) { $clientclass = self::get_client_classname_from_fs($filesystem); $client = new $clientclass(null); if ($client && $client->get_availability()) { - $result[$filesystem] = $filesystem; + $result[$filesystem] = $name; } } return $result; diff --git a/classes/local/store/azure/client.php b/classes/local/store/azure/client.php index 047b07d7..e8e4076e 100644 --- a/classes/local/store/azure/client.php +++ b/classes/local/store/azure/client.php @@ -32,6 +32,7 @@ /** * client + * @deprecated Since Moodle 4.2 - Please see the README about updating to new azure_blob_storage client. */ class client extends object_client_base { diff --git a/classes/local/store/azure/file_system.php b/classes/local/store/azure/file_system.php index aa91803a..ef5847e3 100644 --- a/classes/local/store/azure/file_system.php +++ b/classes/local/store/azure/file_system.php @@ -33,6 +33,7 @@ /** * file_system + * @deprecated Since Moodle 4.2 - Please see the README about updating to new azure_blob_storage client. */ class file_system extends object_file_system { diff --git a/classes/local/store/azure/stream_wrapper.php b/classes/local/store/azure/stream_wrapper.php index 47a67a87..fed9e986 100644 --- a/classes/local/store/azure/stream_wrapper.php +++ b/classes/local/store/azure/stream_wrapper.php @@ -44,6 +44,7 @@ /** * stream_wrapper + * @deprecated Since Moodle 4.2 - Please see the README about updating to new azure_blob_storage client. */ class stream_wrapper { diff --git a/classes/local/store/azure_blob_storage/client.php b/classes/local/store/azure_blob_storage/client.php new file mode 100644 index 00000000..2aeb5a25 --- /dev/null +++ b/classes/local/store/azure_blob_storage/client.php @@ -0,0 +1,296 @@ +. + +namespace tool_objectfs\local\store\azure_blob_storage; + +use admin_settingpage; +use coding_exception; +use GuzzleHttp\Psr7\Utils; +use tool_objectfs\local\store\object_client_base; +use local_azureblobstorage\api; +use local_azureblobstorage\stream_wrapper; +use stdClass; +use Throwable; + +/** + * Azure blob storage client + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright 2024 Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class client extends object_client_base { + + /** @var api $api Azure API */ + protected api $api; + + /** + * Creates object client + * @param stdClass $config / TODO is this maybe null ? + */ + public function __construct($config) { + if (empty($config) || !$this->get_availability()) { + parent::__construct($config); + return; + } + + $this->api = new api($config->azure_accountname, $config->azure_container, $config->azure_sastoken); + $this->config = $config; + $this->maxupload = api::MAX_BLOCK_SIZE; + } + + /** + * Determines if this filesystem is available for use. + * @return bool + */ + public function get_availability(): bool { + // Requires local_azureblobstorage to be installed. + // Namespace changed in 4.4+. + if (class_exists('\core\plugin_manager')) { + $info = \core\plugin_manager::instance()->get_plugin_info('local_azureblobstorage'); + // For 4.2 and 4.3. + } else if (class_exists('\core_plugin_manager')) { + $info = \core_plugin_manager::instance()->get_plugin_info('local_azureblobstorage'); + } else { + throw new coding_exception("Could not load plugin manager class"); + } + + // Info is empty if plugin is not installed or no API is setup (missing config?). + return !empty($info); + } + + /** + * Sets the StreamWrapper to allow accessing the remote content via a blob:// path. + */ + public function register_stream_wrapper() { + if ($this->get_availability()) { + stream_wrapper::register($this->api); + } else { + parent::register_stream_wrapper(); + } + } + + /** + * Returns the full path for a given file by contenthash + * @param string $contenthash + * @return string filepath + */ + public function get_fullpath_from_hash($contenthash): string { + $filepath = $this->get_filepath_from_hash($contenthash); + return "blob://$filepath"; + } + + /** + * Returns the filepath from the contenthash, mimicking the + * structure of the filedir storage system. + * @param string $contenthash + * @return string filepath + */ + protected function get_filepath_from_hash($contenthash): string { + $l1 = $contenthash[0] . $contenthash[1]; + $l2 = $contenthash[2] . $contenthash[3]; + return "$l1/$l2/$contenthash"; + } + + /** + * Returns the blob key (the key used to reference the blob) from a given filepath. + * @param string $filepath + * @return string + */ + protected function get_blob_key_from_path(string $filepath): string { + return str_replace("blob://", '', $filepath); + } + + /** + * Deletes a given file + * @param string $fullpath + */ + public function delete_file($fullpath) { + // Stream wrapper supports unlinking, so just unlink. + unlink($fullpath); + } + + /** + * Renames a given file + * @param string $currentpath + * @param string $destinationpath + */ + public function rename_file($currentpath, $destinationpath) { + // Azure does not support renaming, instead the file is copied + // and the old one is deleted. + copy($currentpath, $destinationpath); + $this->delete_file($currentpath); + } + + /** + * Verifies an object is uploaded correctly. + * In Azure, this is done by checking the md5 hash of the contents. + * @param string $contenthash + * @param string $localpath + * @return bool + */ + public function verify_object($contenthash, $localpath) { + // If the object is uploaded to Azure the content will always be correct, + // because Azure will reject the original upload request if the md5 given during + // upload does not match. + // So here we just check the blob exists, and don't actually care about comparing the md5. + try { + // Just query the properties to confirm the file does indeed exist. + $key = $this->get_filepath_from_hash($contenthash); + $this->api->get_blob_properties_async($key)->wait(); + return true; + } catch (Throwable $e) { + return false; + } + } + + /** + * Returns a stream context used to handle file IO + * @return resource stream resource + */ + public function get_seekable_stream_context() { + $context = stream_context_create([ + 'blob' => [ + 'seekable' => true, + ], + ]); + return $context; + } + + /** + * Test permissions by uploading and doing various actions. + * @param bool $testdelete if should test deletion. + * @return stdClass containing 'success' and 'messages' values. + */ + public function test_permissions($testdelete): stdClass { + $key = 'permissions_check_test'; + $file = Utils::streamFor('test permission file'); + $filemd5 = hex2bin(md5('test permission file')); + + // Try create a file. + try { + $this->api->put_blob_async($key, $file, $filemd5)->wait(); + } catch (Throwable $e) { + return (object) [ + 'success' => false, + 'messages' => [get_string('settings:writefailure', 'tool_objectfs') . $e->getMessage() => 'notifyproblem'], + ]; + } + + // Try read the file that was created. + try { + $this->api->get_blob_async($key, $file, $filemd5)->wait(); + } catch (Throwable $e) { + return (object) [ + 'success' => false, + 'messages' => [get_string('settings:permissionreadfailure', 'tool_objectfs') . $e->getMessage() => 'notifyproblem'], + ]; + } + + // If testing delete, try delete the test file. + if ($testdelete) { + try { + $this->api->delete_blob_async($key)->wait(); + } catch (Throwable $e) { + return (object) [ + 'success' => false, + 'messages' => [get_string('settings:deleteerror', 'tool_objectfs') . $e->getMessage() => 'notifyproblem'], + ]; + } + } + + return (object) [ + 'success' => true, + 'messages' => [get_string('settings:permissioncheckpassed', 'tool_objectfs') => 'notifysuccess'], + ]; + } + + /** + * Tests connection + * @return stdClass with 'success' and 'details' values. + */ + public function test_connection(): stdClass { + // Try to create a file. + try { + $this->api->put_blob_async('connection_check_test', Utils::streamFor('test contents'), hex2bin(md5('test contents'))); + } catch (Throwable $e) { + return (object) [ + 'success' => false, + 'details' => $e->getMessage(), + ]; + } + + return (object) [ + 'success' => true, + 'details' => '', + ]; + } + + /** + * Returns token expiry time + * @return int + */ + public function get_token_expiry_time(): int { + if (empty($this->config->azure_sastoken)) { + return -1; + } + + // Parse the sas token (it just uses url parameter encoding). + $parts = []; + parse_str($this->config->azure_sastoken, $parts); + + // Get the 'se' part (signed expiry). + if (!isset($parts['se'])) { + // Assume expired (malformed). + return 0; + } + + // Parse timestamp string into unix timestamp int. + $expirystr = $parts['se']; + return strtotime($expirystr); + } + + /** + * Azure settings form with the following elements: + * + * Storage account name. + * Container name. + * Shared Access Signature. + * + * @param admin_settingpage $settings + * @param \stdClass $config + * @return admin_settingpage + */ + public function define_client_section($settings, $config): admin_settingpage { + $settings->add(new \admin_setting_heading('tool_objectfs/azure', + new \lang_string('settings:azure:header', 'tool_objectfs'), $this->define_client_check())); + + $settings->add(new \admin_setting_configtext('tool_objectfs/azure_accountname', + new \lang_string('settings:azure:accountname', 'tool_objectfs'), + new \lang_string('settings:azure:accountname_help', 'tool_objectfs'), '')); + + $settings->add(new \admin_setting_configtext('tool_objectfs/azure_container', + new \lang_string('settings:azure:container', 'tool_objectfs'), + new \lang_string('settings:azure:container_help', 'tool_objectfs'), '')); + + $settings->add(new \admin_setting_configpasswordunmask('tool_objectfs/azure_sastoken', + new \lang_string('settings:azure:sastoken', 'tool_objectfs'), + new \lang_string('settings:azure:sastoken_help', 'tool_objectfs'), '')); + + return $settings; + } +} diff --git a/classes/local/store/azure_blob_storage/file_system.php b/classes/local/store/azure_blob_storage/file_system.php new file mode 100644 index 00000000..275d58e2 --- /dev/null +++ b/classes/local/store/azure_blob_storage/file_system.php @@ -0,0 +1,39 @@ +. + +namespace tool_objectfs\local\store\azure_blob_storage; + +use tool_objectfs\local\store\azure_blob_storage\client; +use tool_objectfs\local\store\object_file_system; + +/** + * Azure blob store file system + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright 2024 Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class file_system extends object_file_system { + /** + * Initialise client + * @param mixed $config + * @return client + */ + protected function initialise_external_client($config) { + return new client($config); + } +} diff --git a/classes/tests/test_azure_blob_storage_integration_client.php b/classes/tests/test_azure_blob_storage_integration_client.php new file mode 100644 index 00000000..6f8804b8 --- /dev/null +++ b/classes/tests/test_azure_blob_storage_integration_client.php @@ -0,0 +1,60 @@ +. + +namespace tool_objectfs\tests; + +use tool_objectfs\local\store\azure_blob_storage\client; + +/** + * Client used for integration testing azure blob storage client + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright 2024 Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class test_azure_blob_storage_integration_client extends client { + /** + * @var string + */ + private $runidentifier; + + /** + * construct + * @param mixed $config + * @return void + */ + public function __construct($config) { + parent::__construct($config); + $time = microtime(); + $this->runidentifier = md5($time); + } + + /** + * get_filepath_from_hash + * @param mixed $contenthash + * + * @return string + */ + protected function get_filepath_from_hash($contenthash): string { + $l1 = $contenthash[0] . $contenthash[1]; + $l2 = $contenthash[2] . $contenthash[3]; + $runidentifier = $this->runidentifier; + return "test/$runidentifier/$l1/$l2/$contenthash"; + } + +} + diff --git a/classes/tests/test_azure_integration_client.php b/classes/tests/test_azure_integration_client.php index c871e1f4..9f48b209 100644 --- a/classes/tests/test_azure_integration_client.php +++ b/classes/tests/test_azure_integration_client.php @@ -24,6 +24,7 @@ * @package tool_objectfs * @copyright Catalyst IT * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @deprecated Since Moodle 4.2 - Please see the README about updating to new azure_blob_storage client. */ class test_azure_integration_client extends client { diff --git a/classes/tests/test_file_system.php b/classes/tests/test_file_system.php index 7e1ba875..2a938035 100644 --- a/classes/tests/test_file_system.php +++ b/classes/tests/test_file_system.php @@ -64,6 +64,13 @@ protected function initialise_external_client($config) { $config->azure_sastoken = $credentials['azure_sastoken']; manager::set_objectfs_config($config); $client = new test_azure_integration_client($config); + } else if (isset($CFG->phpunit_objectfs_azure_blob_storage_integration_test_credentials)) { + $credentials = $CFG->phpunit_objectfs_azure_integration_test_credentials; + $config->azure_accountname = $credentials['azure_accountname']; + $config->azure_container = $credentials['azure_container']; + $config->azure_sastoken = $credentials['azure_sastoken']; + manager::set_objectfs_config($config); + $client = new test_azure_blob_storage_integration_client($config); } else if (isset($CFG->phpunit_objectfs_swift_integration_test_credentials)) { $credentials = $CFG->phpunit_objectfs_swift_integration_test_credentials; $config->openstack_authurl = $credentials['openstack_authurl']; diff --git a/tests/object_file_system_test.php b/tests/object_file_system_test.php index 6843bd6b..0abe42a7 100644 --- a/tests/object_file_system_test.php +++ b/tests/object_file_system_test.php @@ -945,6 +945,13 @@ public function test_is_configured_fake_autoloader() { $autoloaderref = $clientref->getParentClass()->getProperty('autoloader'); $autoloaderref->setAccessible(true); $autoloader = $autoloaderref->getValue($this->filesystem->externalclient); + + // If client does not have autoloader, skip test. + if (empty($autoloader)) { + $this->markTestSkipped("Client does not have autoloader"); + return; + } + $this->set_externalclient_config('autoloader', $autoloader . '_fake'); $this->assertFalse($this->filesystem->is_configured()); } From e54974546ae0f568fb51e1d23f7c39f7bc4e64f8 Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Thu, 7 Nov 2024 12:20:23 +1000 Subject: [PATCH 19/28] docs: update azure blob permissions docs --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 348e68e4..8b04098e 100644 --- a/README.md +++ b/README.md @@ -209,7 +209,9 @@ az storage container policy create \ --name \ --start \ --expiry \ - --permissions rw + --permissions racwl + // Or optionally to allow delete + --permissions racwld # Start and Expiry are optional arguments. ``` From 04756bceb526ed82ec3e87f1f889d0d7db2c3833 Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Thu, 7 Nov 2024 12:20:42 +1000 Subject: [PATCH 20/28] feat: ensure file copied before deleting during rename --- classes/local/store/azure_blob_storage/client.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/classes/local/store/azure_blob_storage/client.php b/classes/local/store/azure_blob_storage/client.php index 2aeb5a25..3668ccf4 100644 --- a/classes/local/store/azure_blob_storage/client.php +++ b/classes/local/store/azure_blob_storage/client.php @@ -18,6 +18,7 @@ use admin_settingpage; use coding_exception; +use Exception; use GuzzleHttp\Psr7\Utils; use tool_objectfs\local\store\object_client_base; use local_azureblobstorage\api; @@ -133,6 +134,12 @@ public function rename_file($currentpath, $destinationpath) { // Azure does not support renaming, instead the file is copied // and the old one is deleted. copy($currentpath, $destinationpath); + + // Ensure file exists as a fail-safe. + if (!is_readable($destinationpath)) { + throw new Exception("Rename (copy and delete) failed because copy operation failed, skipping deletion."); + } + $this->delete_file($currentpath); } From 7d74259df0955758054a2376789b900090032483 Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Thu, 7 Nov 2024 12:21:05 +1000 Subject: [PATCH 21/28] tweak: use azure:// instead of blob:// --- classes/local/store/azure_blob_storage/client.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/classes/local/store/azure_blob_storage/client.php b/classes/local/store/azure_blob_storage/client.php index 3668ccf4..2c3fa327 100644 --- a/classes/local/store/azure_blob_storage/client.php +++ b/classes/local/store/azure_blob_storage/client.php @@ -92,7 +92,7 @@ public function register_stream_wrapper() { */ public function get_fullpath_from_hash($contenthash): string { $filepath = $this->get_filepath_from_hash($contenthash); - return "blob://$filepath"; + return "azure://$filepath"; } /** @@ -113,7 +113,7 @@ protected function get_filepath_from_hash($contenthash): string { * @return string */ protected function get_blob_key_from_path(string $filepath): string { - return str_replace("blob://", '', $filepath); + return str_replace("azure://", '', $filepath); } /** @@ -171,7 +171,7 @@ public function verify_object($contenthash, $localpath) { */ public function get_seekable_stream_context() { $context = stream_context_create([ - 'blob' => [ + 'azure' => [ 'seekable' => true, ], ]); From 15baa7704161a6812b72c9035e9c5187f759d3ab Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Thu, 7 Nov 2024 12:21:28 +1000 Subject: [PATCH 22/28] tweak: move credential expiry logic inside of api --- classes/local/store/azure_blob_storage/client.php | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/classes/local/store/azure_blob_storage/client.php b/classes/local/store/azure_blob_storage/client.php index 2c3fa327..bffc05d3 100644 --- a/classes/local/store/azure_blob_storage/client.php +++ b/classes/local/store/azure_blob_storage/client.php @@ -256,19 +256,14 @@ public function get_token_expiry_time(): int { return -1; } - // Parse the sas token (it just uses url parameter encoding). - $parts = []; - parse_str($this->config->azure_sastoken, $parts); + // Return expiry time, or default to 0 if could not parse. + $time = $this->api->get_token_expiry_time(); - // Get the 'se' part (signed expiry). - if (!isset($parts['se'])) { - // Assume expired (malformed). + if (is_null($time)) { return 0; } - // Parse timestamp string into unix timestamp int. - $expirystr = $parts['se']; - return strtotime($expirystr); + return $time; } /** From d51e58b42161ce80d405edf7898f7b33abc72c04 Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Thu, 7 Nov 2024 12:21:42 +1000 Subject: [PATCH 23/28] bugfix: unit test credentials reference --- classes/tests/test_file_system.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/tests/test_file_system.php b/classes/tests/test_file_system.php index 2a938035..5b51d8f7 100644 --- a/classes/tests/test_file_system.php +++ b/classes/tests/test_file_system.php @@ -65,7 +65,7 @@ protected function initialise_external_client($config) { manager::set_objectfs_config($config); $client = new test_azure_integration_client($config); } else if (isset($CFG->phpunit_objectfs_azure_blob_storage_integration_test_credentials)) { - $credentials = $CFG->phpunit_objectfs_azure_integration_test_credentials; + $credentials = $CFG->phpunit_objectfs_azure_blob_storage_integration_test_credentials; $config->azure_accountname = $credentials['azure_accountname']; $config->azure_container = $credentials['azure_container']; $config->azure_sastoken = $credentials['azure_sastoken']; From 3d8232fddc2c324bd6c6a419e8c49528bbc4be9f Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Fri, 8 Nov 2024 10:26:37 +1000 Subject: [PATCH 24/28] chore: version bump --- version.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version.php b/version.php index 09084a53..ade329a6 100644 --- a/version.php +++ b/version.php @@ -25,8 +25,8 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2024102400; // The current plugin version (Date: YYYYMMDDXX). -$plugin->release = 2024102400; // Same as version. +$plugin->version = 2024110800; // The current plugin version (Date: YYYYMMDDXX). +$plugin->release = 2024110800; // Same as version. $plugin->requires = 2023042400; // Requires 4.2. $plugin->component = "tool_objectfs"; $plugin->maturity = MATURITY_STABLE; From 3467fb0e1f06cfd52de85716aa6810b84518ca98 Mon Sep 17 00:00:00 2001 From: Dan Marsden Date: Fri, 22 Nov 2024 11:20:42 +1300 Subject: [PATCH 25/28] remove old branches that we don't officially support anymore. --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 8b04098e..9e1ce76e 100644 --- a/README.md +++ b/README.md @@ -73,15 +73,12 @@ https://github.com/catalyst/moodle-local_datacleaner This plugin is GDPR complient if you enable the deletion of remote objects. -## Branches +## Supported branches | Moodle version | Totara version | Branch | PHP | MySQL | PostgreSQL | |-------------------|--------------------------|----------------------------------------------------------------------------------------------|------|---------|-------------| | Moodle 4.2+ | | [MOODLE_402_STABLE](https://github.com/catalyst/moodle-tool_objectfs/tree/MOODLE_402_STABLE) | 8.0+ | 8.0+ | 13+ | | Moodle 3.10 - 4.1 | | [MOODLE_310_STABLE](https://github.com/catalyst/moodle-tool_objectfs/tree/MOODLE_310_STABLE) | 7.2+ | 5.7+ | 12+ | -| Moodle 3.3 - 3.9 | Totara 12 | [MOODLE_33_STABLE](https://github.com/catalyst/moodle-tool_objectfs/tree/MOODLE_33_STABLE) | 7.1+ | 5.6+ | 9.5+ | -| Moodle 2.7 - 3.2 | Totara 2.7 - 2.9, 9 - 11 | [27-32-STABLE](https://github.com/catalyst/moodle-tool_objectfs/tree/27-32-STABLE) | 5.5+ | 5.5.31+ | 9.1+ | - ## Installation 1. If not on Moodle 3.3, backport the file system API. See [Backporting](#backporting) From 883f210b901b60dee38765120383e971929d92af Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Fri, 6 Dec 2024 14:05:51 +1000 Subject: [PATCH 26/28] Add object tagging - Moodle 310 (#619) (#623) * feat: add object tagging * test: fix unit test count checking * tagging: don't wait for object lock * tagging: improve migration controls and progress visibility * feat: move tagging status reports to check api * feat: display header with status report * refactor: integrate tagpushedtime into single update query * refactor: store tags against object id instead of hash * chore: organise tagging lang strings * bugfix: fix mysql query compatibility * tagging: move mimetype to metadata, add location/orphan tag source * tagging: check environment config length * settings: use admin_setting_check if available * report: remove object size from tag count report * tagging: ignore if cannot get lock * ci: small fixups * tagging: switch to admin setting for tagging environment * refactor: get object tag sync status count details separately * refactor: tweak defaults and add tagging adhoc task spawn limit * tests: reset static file storage before tests * bugfix: fix test * ci: fixup --- TAGGING.md | 65 +++ classes/check/tagging_migration_status.php | 80 ++++ classes/check/tagging_status.php | 62 +++ classes/check/tagging_sync_status.php | 60 +++ classes/local/manager.php | 25 +- .../local/object_manipulator/manipulator.php | 2 +- .../report/object_status_history_table.php | 5 + classes/local/report/objectfs_report.php | 4 +- .../local/report/tag_count_report_builder.php | 48 ++ classes/local/store/object_client.php | 28 ++ classes/local/store/object_client_base.php | 36 ++ classes/local/store/object_file_system.php | 123 +++++ classes/local/store/s3/client.php | 123 ++++- classes/local/store/s3/file_system.php | 3 +- classes/local/tag/environment_source.php | 73 +++ classes/local/tag/location_source.php | 57 +++ classes/local/tag/tag_manager.php | 243 ++++++++++ classes/local/tag/tag_source.php | 50 ++ classes/local/tag_sync_count_result.php | 55 +++ classes/task/trigger_update_object_tags.php | 55 +++ classes/task/update_object_tags.php | 124 +++++ classes/tests/test_client.php | 37 ++ classes/tests/testcase.php | 25 +- db/install.xml | 20 +- db/tasks.php | 12 + db/upgrade.php | 53 +++ lang/en/tool_objectfs.php | 59 +++ lib.php | 4 + object_status.php | 3 + settings.php | 72 ++- tests/check/tagging_migration_status_test.php | 73 +++ tests/check/tagging_sync_status_test.php | 68 +++ tests/local/report/object_status_test.php | 2 +- tests/local/tagging_test.php | 442 ++++++++++++++++++ tests/object_file_system_test.php | 146 ++++++ tests/task/populate_objects_filesize_test.php | 13 +- .../task/trigger_update_object_tags_test.php | 50 ++ tests/task/update_object_tags_test.php | 204 ++++++++ version.php | 4 +- 39 files changed, 2581 insertions(+), 27 deletions(-) create mode 100644 TAGGING.md create mode 100644 classes/check/tagging_migration_status.php create mode 100644 classes/check/tagging_status.php create mode 100644 classes/check/tagging_sync_status.php create mode 100644 classes/local/report/tag_count_report_builder.php create mode 100644 classes/local/tag/environment_source.php create mode 100644 classes/local/tag/location_source.php create mode 100644 classes/local/tag/tag_manager.php create mode 100644 classes/local/tag/tag_source.php create mode 100644 classes/local/tag_sync_count_result.php create mode 100644 classes/task/trigger_update_object_tags.php create mode 100644 classes/task/update_object_tags.php create mode 100644 tests/check/tagging_migration_status_test.php create mode 100644 tests/check/tagging_sync_status_test.php create mode 100644 tests/local/tagging_test.php create mode 100644 tests/task/trigger_update_object_tags_test.php create mode 100644 tests/task/update_object_tags_test.php diff --git a/TAGGING.md b/TAGGING.md new file mode 100644 index 00000000..4ceae4a7 --- /dev/null +++ b/TAGGING.md @@ -0,0 +1,65 @@ +# Tagging +Tagging allows extra metadata about your files to be send to the external object store. These sources are defined in code, and currently cannot be configured on/off from the UI. + +Currently, this is only implemented for the S3 file system client. +**Tagging vs metadata** + +Note object tags are different from object metadata. + +Object metadata is immutable, and attached to the object on upload. With metadata, if you wish to update it (for example during a migration, or the sources changed), you have to copy the object with the new metadata, and delete the old object. This is not ideal, since deletion is optional in objectfs. + +Object tags are more suitable, since their permissions can be managed separately (e.g. a client can be allowed to modify tags, but not delete objects). + +## File system setup +### S3 +[See the S3 docs for more information about tagging](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html). + +You must allow `s3:GetObjectTagging` and `s3:PutObjectTagging` permission to the objectfs client. + +## Sources +The following sources are implemented currently: +### Environment +What environment the file was uploaded in. Configure the environment using `taggingenvironment` in the objectfs plugin settings. + +This tag is also used by objectfs to determine if tags can be overwritten. See [Multiple environments setup](#multiple-environments-setup) for more information. + +### Location +Either `orphan` if the file no longer exists in the `files` table in Moodle, otherwise `active`. + +## Multiple environments setup +This feature is designed to work in situations where multiple environments (e.g. prod, staging) points to the same bucket, however, some setup is needed: + +1. Turn off `overwriteobjecttags` in every environment except the production environment. +2. Configure `taggingenvironment` to be unique for all environments. + +By doing the above two steps, it will allow the production environment to always set its own tags, even if a file was first uploaded to staging and then to production. + +Lower environments can still update tags, but only if the `environment` matches theirs. This allows staging to manage object tags on objects only it knows about, but as soon as the file is uploaded from production (and therefore have it's environment tag replaced with `prod`), staging will no longer touch it. + +## Migration +Only new objects uploaded after enabling this feature will have tags added. To backfill tags for previously uploaded objects, you must do the following: + +- Manually run `trigger_update_object_tags` scheduled task from the UI, which queues a `update_object_tags` adhoc task that will process all objects marked as needing sync. +or +- Call the CLI to execute a `update_object_tags` adhoc task manually. + +You may need to update the DB to mark objects tag sync status as needing sync if the object has previously been synced before. +## Reporting +There is an additional graph added to the object summary report showing the tag value combinations and counts of each. + +Note, this is only for files that have been uploaded from the respective environment, and may not be consistent for environments where `overwriteobjecttags` is disabled (because the site does not know if a file was overwritten in the external store by another client). + +## For developers + +### Adding a new source +Note the rules about sources: +- Identifier must be < 32 chars long. +- Value must be < 128 chars long. + +While external providers allow longer key/values, we intentionally limit it to reserve space for future use. These limits may change in the future as the feature matures. + +To add a new source: +- Implement `tag_source` +- Add to the `tag_manager` class +- As part of an upgrade step, mark all objects `tagsyncstatus` to needing sync (using `tag_manager` class, or manually in the DB) +- As part of an upgrade step, queue a `update_object_tags` adhoc task to process the tag migration. \ No newline at end of file diff --git a/classes/check/tagging_migration_status.php b/classes/check/tagging_migration_status.php new file mode 100644 index 00000000..abaa9de4 --- /dev/null +++ b/classes/check/tagging_migration_status.php @@ -0,0 +1,80 @@ +. + +namespace tool_objectfs\check; + +use core\check\check; +use core\check\result; +use core\task\manager; +use html_table; +use html_writer; +use tool_objectfs\task\update_object_tags; + +/** + * Tagging migration status check + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class tagging_migration_status extends check { + /** + * Link to ObjectFS settings page. + * + * @return \action_link|null + */ + public function get_action_link(): ?\action_link { + $url = new \moodle_url('/admin/category.php', ['category' => 'tool_objectfs']); + return new \action_link($url, get_string('pluginname', 'tool_objectfs')); + } + + /** + * Get result + * @return result + */ + public function get_result(): result { + // We want to check this regardless if enabled or supported and not exit early. + // Because it may have been turned off accidentally thus causing the migration to fail. + $tasks = manager::get_adhoc_tasks(update_object_tags::class); + + if (empty($tasks)) { + return new result(result::NA, get_string('tagging:migration:nothingrunning', 'tool_objectfs')); + } + + $table = new html_table(); + $table->head = [ + get_string('table:taskid', 'tool_objectfs'), + get_string('table:iteration', 'tool_objectfs'), + get_string('table:status', 'tool_objectfs'), + ]; + + foreach ($tasks as $task) { + $table->data[$task->get_id()] = [$task->get_id(), $task->get_iteration(), $task->get_status_badge()]; + } + $html = html_writer::table($table); + + $ataskisfailing = !empty(array_filter($tasks, function($task) { + return $task->get_fail_delay() > 0; + })); + + if ($ataskisfailing) { + return new result(result::WARNING, get_string('check:tagging:migrationerror', 'tool_objectfs'), $html); + } + + return new result(result::OK, get_string('check:tagging:migrationok', 'tool_objectfs'), $html); + } +} diff --git a/classes/check/tagging_status.php b/classes/check/tagging_status.php new file mode 100644 index 00000000..df3e68d5 --- /dev/null +++ b/classes/check/tagging_status.php @@ -0,0 +1,62 @@ +. + +namespace tool_objectfs\check; + +use core\check\check; +use core\check\result; +use tool_objectfs\local\tag\tag_manager; + +/** + * Tagging status check + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class tagging_status extends check { + /** + * Link to ObjectFS settings page. + * + * @return \action_link|null + */ + public function get_action_link(): ?\action_link { + $url = new \moodle_url('/admin/category.php', ['category' => 'tool_objectfs']); + return new \action_link($url, get_string('pluginname', 'tool_objectfs')); + } + + /** + * Get result + * @return result + */ + public function get_result(): result { + if (!tag_manager::is_tagging_enabled_and_supported()) { + return new result(result::NA, get_string('check:tagging:na', 'tool_objectfs')); + } + + // Do a tag set test. + $config = \tool_objectfs\local\manager::get_objectfs_config(); + $client = \tool_objectfs\local\manager::get_client($config); + $result = $client->test_set_object_tag(); + + if ($result->success) { + return new result(result::OK, get_string('check:tagging:ok', 'tool_objectfs'), $result->details); + } else { + return new result(result::ERROR, get_string('check:tagging:error', 'tool_objectfs'), $result->details); + } + } +} diff --git a/classes/check/tagging_sync_status.php b/classes/check/tagging_sync_status.php new file mode 100644 index 00000000..0cc12e45 --- /dev/null +++ b/classes/check/tagging_sync_status.php @@ -0,0 +1,60 @@ +. + +namespace tool_objectfs\check; + +use core\check\check; +use core\check\result; +use tool_objectfs\local\tag\tag_manager; +use tool_objectfs\local\tag_sync_count_result; + +/** + * Tagging sync status check + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class tagging_sync_status extends check { + /** + * Link to ObjectFS settings page. + * + * @return \action_link|null + */ + public function get_action_link(): ?\action_link { + $url = new \moodle_url('/admin/category.php', ['category' => 'tool_objectfs']); + return new \action_link($url, get_string('pluginname', 'tool_objectfs')); + } + + /** + * Get result + * @return result + */ + public function get_result(): result { + if (!tag_manager::is_tagging_enabled_and_supported()) { + return new tag_sync_count_result(result::NA, get_string('check:tagging:na', 'tool_objectfs')); + } + + // We only do a lightweight check here, the get_details is overwritten in tag_sync_status_result + // to provide more information that is more computationally expensive to calculate. + if (tag_manager::tag_sync_errors_exist()) { + return new tag_sync_count_result(result::WARNING, get_string('check:tagging:syncerror', 'tool_objectfs')); + } + + return new tag_sync_count_result(result::OK, get_string('check:tagging:syncok', 'tool_objectfs')); + } +} diff --git a/classes/local/manager.php b/classes/local/manager.php index ae763fda..67b89406 100644 --- a/classes/local/manager.php +++ b/classes/local/manager.php @@ -27,6 +27,7 @@ use stdClass; use tool_objectfs\local\store\object_file_system; +use tool_objectfs\local\tag\tag_manager; /** * [Description manager] @@ -64,6 +65,7 @@ public static function get_objectfs_config() { $config->batchsize = 10000; $config->useproxy = 0; $config->deleteexternal = 0; + $config->enabletagging = false; $config->filesystem = ''; $config->enablepresignedurls = 0; @@ -159,7 +161,7 @@ public static function update_object_by_hash($contenthash, $newlocation, $filesi $newobject->filesize = isset($oldobject->filesize) ? $oldobject->filesize : $DB->get_field('files', 'filesize', ['contenthash' => $contenthash], IGNORE_MULTIPLE); - return self::update_object($newobject, $newlocation); + return self::upsert_object($newobject, $newlocation); } $newobject->location = $newlocation; @@ -172,9 +174,7 @@ public static function update_object_by_hash($contenthash, $newlocation, $filesi $newobject->filesize = $filesize; $newobject->timeduplicated = time(); } - $DB->insert_record('tool_objectfs_objects', $newobject); - - return $newobject; + return self::upsert_object($newobject, $newlocation); } /** @@ -184,7 +184,7 @@ public static function update_object_by_hash($contenthash, $newlocation, $filesi * @return stdClass * @throws \dml_exception */ - public static function update_object(stdClass $object, $newlocation) { + public static function upsert_object(stdClass $object, $newlocation) { global $DB; // If location change is 'duplicated' we update timeduplicated. @@ -192,8 +192,21 @@ public static function update_object(stdClass $object, $newlocation) { $object->timeduplicated = time(); } + $locationchanged = !isset($object->location) || $object->location != $newlocation; $object->location = $newlocation; - $DB->update_record('tool_objectfs_objects', $object); + + // If id is set, update, else insert new. + if (empty($object->id)) { + $object->id = $DB->insert_record('tool_objectfs_objects', $object); + } else { + $DB->update_record('tool_objectfs_objects', $object); + } + + // Post update, notify tag manager since the location tag likely needs changing. + if ($locationchanged && tag_manager::is_tagging_enabled_and_supported()) { + $fs = get_file_storage()->get_file_system(); + $fs->push_object_tags($object->contenthash); + } return $object; } diff --git a/classes/local/object_manipulator/manipulator.php b/classes/local/object_manipulator/manipulator.php index f5108305..afd21808 100644 --- a/classes/local/object_manipulator/manipulator.php +++ b/classes/local/object_manipulator/manipulator.php @@ -111,7 +111,7 @@ public function execute(array $objectrecords) { $newlocation = $this->manipulate_object($objectrecord); if (!empty($objectrecord->id)) { - manager::update_object($objectrecord, $newlocation); + manager::upsert_object($objectrecord, $newlocation); } else { manager::update_object_by_hash($objectrecord->contenthash, $newlocation); } diff --git a/classes/local/report/object_status_history_table.php b/classes/local/report/object_status_history_table.php index 906689ce..3a2bfc87 100644 --- a/classes/local/report/object_status_history_table.php +++ b/classes/local/report/object_status_history_table.php @@ -74,6 +74,11 @@ public function __construct($reporttype, $reportid) { $columnheaders['runningsize'] = get_string('object_status:runningsize', 'tool_objectfs'); } + // Tag count report does not display the size. + if ($this->reporttype == 'tag_count') { + unset($columnheaders['size']); + } + $this->set_attribute('class', 'table-sm'); $this->define_columns(array_keys($columnheaders)); $this->define_headers(array_values($columnheaders)); diff --git a/classes/local/report/objectfs_report.php b/classes/local/report/objectfs_report.php index cc9eb910..468e35db 100644 --- a/classes/local/report/objectfs_report.php +++ b/classes/local/report/objectfs_report.php @@ -78,7 +78,8 @@ public function add_row($datakey, $objectcount, $objectsum) { */ public function add_rows(array $rows) { foreach ($rows as $row) { - $this->add_row($row->datakey, $row->objectcount, $row->objectsum); + // Note objectsum is optional. + $this->add_row($row->datakey, $row->objectcount, $row->objectsum ?? 0); } } @@ -166,6 +167,7 @@ public static function get_report_types() { 'location', 'log_size', 'mime_type', + 'tag_count', ]; } diff --git a/classes/local/report/tag_count_report_builder.php b/classes/local/report/tag_count_report_builder.php new file mode 100644 index 00000000..bcc6e80e --- /dev/null +++ b/classes/local/report/tag_count_report_builder.php @@ -0,0 +1,48 @@ +. + +namespace tool_objectfs\local\report; + +/** + * Tag count report builder. + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class tag_count_report_builder extends objectfs_report_builder { + /** + * Builds report + * @param int $reportid + * @return objectfs_report + */ + public function build_report($reportid) { + global $DB; + $report = new objectfs_report('tag_count', $reportid); + + // Returns counts of key:value. + $sql = " + SELECT CONCAT(COALESCE(object_tags.tagkey, '(untagged)'), ': ', COALESCE(object_tags.tagvalue, '')) as datakey, + COUNT(DISTINCT object_tags.objectid) as objectcount + FROM {tool_objectfs_object_tags} object_tags + GROUP BY object_tags.tagkey, object_tags.tagvalue + "; + $result = $DB->get_records_sql($sql); + $report->add_rows($result); + return $report; + } +} diff --git a/classes/local/store/object_client.php b/classes/local/store/object_client.php index cec71d77..ae264195 100644 --- a/classes/local/store/object_client.php +++ b/classes/local/store/object_client.php @@ -16,6 +16,8 @@ namespace tool_objectfs\local\store; +use stdClass; + /** * Objectfs client interface. * @@ -147,6 +149,32 @@ public function get_token_expiry_time(): int; * @return bool */ public function should_test_delete(): bool; + + /** + * Tests setting an objects tag. + * @return stdClass containing 'success' and 'details' properties + */ + public function test_set_object_tag(): stdClass; + + /** + * Set the given objects tags in the external store. + * @param string $contenthash file content hash + * @param array $tags array of key=>value pairs to set as tags. + */ + public function set_object_tags(string $contenthash, array $tags); + + /** + * Returns given objects tags queried from the external store. External object must exist. + * @param string $contenthash file content has + * @return array array of key=>value tag pairs + */ + public function get_object_tags(string $contenthash): array; + + /** + * If the client supports object tagging feature. + * @return bool true if supports, else false + */ + public function supports_object_tagging(): bool; } diff --git a/classes/local/store/object_client_base.php b/classes/local/store/object_client_base.php index 0dabbc17..dfb0995d 100644 --- a/classes/local/store/object_client_base.php +++ b/classes/local/store/object_client_base.php @@ -25,6 +25,8 @@ namespace tool_objectfs\local\store; +use stdClass; + /** * [Description object_client_base] */ @@ -201,4 +203,38 @@ public function get_token_expiry_time(): int { // Returning -1 = not implemented. return -1; } + + /** + * Tests setting an objects tag. + * @return stdClass containing 'success' and 'details' properties + */ + public function test_set_object_tag(): stdClass { + return (object)['success' => false, 'details' => '']; + } + + /** + * Set the given objects tags in the external store. + * @param string $contenthash file content hash + * @param array $tags array of key=>value pairs to set as tags. + */ + public function set_object_tags(string $contenthash, array $tags) { + return []; + } + + /** + * Returns given objects tags queried from the external store. External object must exist. + * @param string $contenthash file content has + * @return array array of key=>value tag pairs + */ + public function get_object_tags(string $contenthash): array { + return []; + } + + /** + * If the client supports object tagging feature. + * @return bool true if supports, else false + */ + public function supports_object_tagging(): bool { + return false; + } } diff --git a/classes/local/store/object_file_system.php b/classes/local/store/object_file_system.php index 38e12a2c..86942af6 100644 --- a/classes/local/store/object_file_system.php +++ b/classes/local/store/object_file_system.php @@ -36,7 +36,11 @@ use stored_file; use file_storage; use BlobRestProxy; +use coding_exception; +use Throwable; use tool_objectfs\local\manager; +use tool_objectfs\local\tag\environment_source; +use tool_objectfs\local\tag\tag_manager; defined('MOODLE_INTERNAL') || die(); @@ -164,6 +168,23 @@ protected function get_local_path_from_hash($contenthash, $fetchifnotfound = fal return $path; } + /** + * Returns mimetype for a given hash + * @param string $contenthash + * @return string mimetype as stored in mdl_files + */ + protected function get_mimetype_from_hash(string $contenthash): string { + global $DB; + + // We limit 1 because multiple files can have the same contenthash. + // However, they all have the same mimetype so it does not matter which one we query. + return $DB->get_field_sql('SELECT mimetype + FROM {files} + WHERE contenthash = :hash + LIMIT 1', + ['hash' => $contenthash]); + } + /** * get_remote_path_from_storedfile * @param \stored_file $file @@ -360,6 +381,12 @@ public function copy_object_from_local_to_external_by_hash($contenthash, $object } } + // If tagging is enabled, ensure tags are synced regardless of if object is local or duplicated, etc... + // The file may exist in external store because it was uploaded by another site, but we may want to put our tags onto it. + if (tag_manager::is_tagging_enabled_and_supported()) { + $this->push_object_tags($contenthash); + } + $this->logger->log_object_move('copy_object_from_local_to_external', $initiallocation, $finallocation, @@ -1154,4 +1181,100 @@ private function update_object(array $result): array { return $result; } + + /** + * Pushes tags to the external store (post upload) for a given hash. + * External client must support tagging. + * + * @param string $contenthash file to sync tags for + * @return bool true if set tags, false if could not get lock. + */ + public function push_object_tags(string $contenthash): bool { + if (!$this->get_external_client()->supports_object_tagging()) { + throw new coding_exception("Cannot sync tags, external client does not support tagging."); + } + + // Get a lock before syncing, to ensure other parts of objectfs are not moving/interacting with this object. + // Don't wait for it, we want to fail fast. + $lock = $this->acquire_object_lock($contenthash, 0); + + // No lock - just skip it. + if (!$lock) { + return false; + } + + try { + $canset = $this->can_set_object_tags($contenthash); + $timepushed = 0; + + if ($canset) { + $tags = tag_manager::gather_object_tags_for_upload($contenthash); + $this->get_external_client()->set_object_tags($contenthash, $tags); + tag_manager::store_tags_locally($contenthash, $tags); + + // Record the time it was actually pushed to the external store + // (i.e. not when it existed already and was skipped). + $timepushed = time(); + } + + // Regardless, it has synced. + tag_manager::mark_object_tag_sync_status($contenthash, tag_manager::SYNC_STATUS_COMPLETE, $timepushed); + } catch (Throwable $e) { + $lock->release(); + + // Mark object as tag sync error, this should stop it re-trying until fixed manually. + tag_manager::mark_object_tag_sync_status($contenthash, tag_manager::SYNC_STATUS_ERROR); + + throw $e; + } + $lock->release(); + return true; + } + + /** + * Returns true if the current env can set the given object's tags. + * + * To set the tags: + * - The object must exist + * - We can overwrite tags (and not care about any existing) + * OR + * - We cannot overwrite tags, and the tags are empty or the environment is the same as ours. + * + * Avoids unnecessarily querying tags as this is an extra api call to the object store. + * + * @param string $contenthash + * @return bool + */ + private function can_set_object_tags(string $contenthash): bool { + $objectexists = $this->is_file_readable_externally_by_hash($contenthash); + + // Object must exist, we cannot set tags on an object that is missing. + if (!$objectexists) { + return false; + } + + // If can overwrite tags, we don't care then about any existing tags. + if (tag_manager::can_overwrite_object_tags()) { + return true; + } + + // Else we need to check the tags are empty, or the env matches ours. + $existingtags = $this->get_external_client()->get_object_tags($contenthash); + + // Not set yet, must be a new object. + if (empty($existingtags) || !isset($existingtags[environment_source::get_identifier()])) { + return true; + } + + $envsource = new environment_source(); + $currentenv = $envsource->get_value_for_contenthash($contenthash); + + // Env is the same as ours, allowed to set. + if ($existingtags[environment_source::get_identifier()] == $currentenv) { + return true; + } + + // Else no match, do not set. + return false; + } } diff --git a/classes/local/store/s3/client.php b/classes/local/store/s3/client.php index 23c495af..74e37a54 100644 --- a/classes/local/store/s3/client.php +++ b/classes/local/store/s3/client.php @@ -25,10 +25,13 @@ namespace tool_objectfs\local\store\s3; +use coding_exception; use tool_objectfs\local\manager; use tool_objectfs\local\store\object_client_base; use tool_objectfs\local\store\signed_url; use local_aws\admin_settings_aws_region; +use stdClass; +use Throwable; define('AWS_API_VERSION', '2006-03-01'); define('AWS_CAN_READ_OBJECT', 0); @@ -493,10 +496,11 @@ public function define_client_section($settings, $config) { * * @param string $localpath Path to a local file. * @param string $contenthash Content hash of the file. + * @param string $mimetype the mimetype of the file being uploaded * * @throws \Exception if fails. */ - public function upload_to_s3($localpath, $contenthash) { + public function upload_to_s3($localpath, $contenthash, string $mimetype) { $filehandle = fopen($localpath, 'rb'); if (!$filehandle) { @@ -508,7 +512,13 @@ public function upload_to_s3($localpath, $contenthash) { $uploader = new \Aws\S3\ObjectUploader( $this->client, $this->bucket, $this->bucketkeyprefix . $externalpath, - $filehandle + $filehandle, + 'private', + [ + 'params' => [ + 'ContentType' => $mimetype, + ], + ] ); $uploader->upload(); fclose($filehandle); @@ -875,4 +885,113 @@ public function test_range_request($filesystem) { } return (object)['result' => false, 'error' => get_string('fixturefilemissing', 'tool_objectfs')]; } + + /** + * Tests setting an objects tag. + * @return stdClass containing 'success' and 'details' properties + */ + public function test_set_object_tag(): stdClass { + try { + // First ensure a test object exists to put tags on. + // Note this will override the existing object if exists. + $key = $this->bucketkeyprefix . 'tagging_check_file'; + $this->client->putObject([ + 'Bucket' => $this->bucket, + 'Key' => $key, + 'Body' => 'test content', + ]); + + // Next try to tag it - this will throw an exception if cannot set + // (for example, because it does not have permissions to). + $this->client->putObjectTagging([ + 'Bucket' => $this->bucket, + 'Key' => $key, + 'Tagging' => [ + 'TagSet' => [ + [ + 'Key' => 'test', + 'Value' => 'test', + ], + ], + ], + ]); + } catch (Throwable $e) { + return (object) [ + 'success' => false, + 'details' => $e->getMessage(), + ]; + } + + // Success - no exceptions thrown. + return (object) ['success' => true, 'details' => '']; + } + + /** + * Convert key=>value to s3 tag format + * @param array $tags + * @return array tags in s3 format. + */ + private function convert_tags_to_s3_format(array $tags): array { + foreach ($tags as $key => $value) { + $s3tags[] = [ + 'Key' => $key, + 'Value' => $value, + ]; + } + return $s3tags; + } + + /** + * Set the given objects tags in the external store. + * @param string $contenthash file content hash + * @param array $tags array of key=>value pairs to set as tags. + */ + public function set_object_tags(string $contenthash, array $tags) { + $objectkey = $this->bucketkeyprefix . $this->get_filepath_from_hash($contenthash); + + // Then put onto object. + $this->client->putObjectTagging([ + 'Bucket' => $this->bucket, + 'Key' => $objectkey, + 'Tagging' => [ + 'TagSet' => $this->convert_tags_to_s3_format($tags), + ], + ]); + } + + /** + * Returns given objects tags queried from the external store. Object must exist. + * @param string $contenthash file content has + * @return array array of key=>value tag pairs + */ + public function get_object_tags(string $contenthash): array { + $objectkey = $this->bucketkeyprefix . $this->get_filepath_from_hash($contenthash); + + // Query from S3. + $result = $this->client->getObjectTagging([ + 'Bucket' => $this->bucket, + 'Key' => $objectkey, + ]); + + // Ensure tags are what we expect, and AWS have not changed the format. + if (!array_key_exists('TagSet', $result->toArray())) { + throw new coding_exception("Unexpected tag format received. Result did not contain a TagSet"); + } + + // Convert from S3 format to key=>value format. + $tagkv = []; + foreach ($result->toArray()['TagSet'] as $tag) { + $tagkv[$tag['Key']] = $tag['Value']; + } + + return $tagkv; + } + + /** + * If the client supports object tagging feature. + * @return bool true if supports, else false + */ + public function supports_object_tagging(): bool { + return true; + } } diff --git a/classes/local/store/s3/file_system.php b/classes/local/store/s3/file_system.php index 93637dd4..b1897d3d 100644 --- a/classes/local/store/s3/file_system.php +++ b/classes/local/store/s3/file_system.php @@ -85,9 +85,10 @@ public function readfile(\stored_file $file) { */ public function copy_from_local_to_external($contenthash) { $localpath = $this->get_local_path_from_hash($contenthash); + $mime = $this->get_mimetype_from_hash($contenthash); try { - $this->get_external_client()->upload_to_s3($localpath, $contenthash); + $this->get_external_client()->upload_to_s3($localpath, $contenthash, $mime); return true; } catch (\Exception $e) { $this->get_logger()->error_log( diff --git a/classes/local/tag/environment_source.php b/classes/local/tag/environment_source.php new file mode 100644 index 00000000..d8625d6e --- /dev/null +++ b/classes/local/tag/environment_source.php @@ -0,0 +1,73 @@ +. + +namespace tool_objectfs\local\tag; + +use moodle_exception; + +/** + * Provides current environment to file. + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class environment_source implements tag_source { + /** + * Identifier used in tagging file. Is the 'key' of the tag. + * @return string + */ + public static function get_identifier(): string { + return 'environment'; + } + + /** + * Description for source displayed in the admin settings. + * @return string + */ + public static function get_description(): string { + return get_string('tagsource:environment', 'tool_objectfs', self::get_env()); + } + + /** + * Returns current env value from $CFG + * @return string|null string if set, else null + */ + private static function get_env(): ?string { + $value = get_config('tool_objectfs', 'taggingenvironment'); + + if (empty($value)) { + return null; + } + + // Must never be greater than 128, unlikely, but we must enforce this. + if (strlen($value) > 128) { + throw new moodle_exception('tagsource:environment:toolong', 'tool_objectfs'); + } + + return $value; + } + + /** + * Returns the tag value for the given file contenthash + * @param string $contenthash + * @return string|null environment value. + */ + public function get_value_for_contenthash(string $contenthash): ?string { + return self::get_env(); + } +} diff --git a/classes/local/tag/location_source.php b/classes/local/tag/location_source.php new file mode 100644 index 00000000..1353a03a --- /dev/null +++ b/classes/local/tag/location_source.php @@ -0,0 +1,57 @@ +. + +namespace tool_objectfs\local\tag; + +/** + * Provides location status for a file. + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class location_source implements tag_source { + /** + * Identifier used in tagging file. Is the 'key' of the tag. + * @return string + */ + public static function get_identifier(): string { + return 'location'; + } + + /** + * Description for source displayed in the admin settings. + * @return string + */ + public static function get_description(): string { + return get_string('tagsource:location', 'tool_objectfs'); + } + + /** + * Returns the tag value for the given file contenthash + * @param string $contenthash + * @return string|null mime type for file. + */ + public function get_value_for_contenthash(string $contenthash): ?string { + global $DB; + + $isorphaned = $DB->record_exists('tool_objectfs_objects', ['contenthash' => $contenthash, + 'location' => OBJECT_LOCATION_ORPHANED]); + + return $isorphaned ? 'orphan' : 'active'; + } +} diff --git a/classes/local/tag/tag_manager.php b/classes/local/tag/tag_manager.php new file mode 100644 index 00000000..603d04a4 --- /dev/null +++ b/classes/local/tag/tag_manager.php @@ -0,0 +1,243 @@ +. + +namespace tool_objectfs\local\tag; + +use coding_exception; +use html_table; +use html_writer; +use tool_objectfs\local\manager; + +defined('MOODLE_INTERNAL') || die(); +require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); + +/** + * Manages object tagging feature. + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class tag_manager { + + /** + * @var int If object needs sync. These will periodically be picked up by scheduled tasks and queued for syncing. + */ + public const SYNC_STATUS_NEEDS_SYNC = 0; + + /** + * @var int Object does not need sync. Will be essentially ignored in tagging process. + */ + public const SYNC_STATUS_COMPLETE = 1; + + /** + * @var int Object tried to sync but there was an error. Will make it ignored and must be corrected manually. + */ + public const SYNC_STATUS_ERROR = 2; + + /** + * @var array All possible tag sync statuses. + */ + public const SYNC_STATUSES = [ + self::SYNC_STATUS_NEEDS_SYNC, + self::SYNC_STATUS_COMPLETE, + self::SYNC_STATUS_ERROR, + ]; + + /** + * Returns an array of tag_source instances that are currently defined. + * @return array + */ + public static function get_defined_tag_sources(): array { + // All possible tag sources should be defined here. + // Note this should be a maximum of 10 sources, as this is an AWS limit. + return [ + new environment_source(), + new location_source(), + ]; + } + + /** + * Is the tagging feature enabled and supported by the configured fs? + * @return bool + */ + public static function is_tagging_enabled_and_supported(): bool { + $enabledinconfig = !empty(get_config('tool_objectfs', 'taggingenabled')); + + $client = manager::get_client(manager::get_objectfs_config()); + $supportedbyfs = !empty($client) && $client->supports_object_tagging(); + + return $enabledinconfig && $supportedbyfs; + } + + /** + * Gathers the tag values for a given content hash + * @param string $contenthash + * @return array array of key=>value pairs, the tags for the given file. + */ + public static function gather_object_tags_for_upload(string $contenthash): array { + $tags = []; + foreach (self::get_defined_tag_sources() as $source) { + $val = $source->get_value_for_contenthash($contenthash); + + // Null means not set for this object. + if (is_null($val)) { + continue; + } + + $tags[$source->get_identifier()] = $val; + } + return $tags; + } + + /** + * Stores tag records for contenthash locally + * @param string $contenthash + * @param array $tags + */ + public static function store_tags_locally(string $contenthash, array $tags) { + global $DB; + + // Lookup object id. + $objectid = $DB->get_field('tool_objectfs_objects', 'id', ['contenthash' => $contenthash], MUST_EXIST); + + // Purge any existing tags for this object. + $DB->delete_records('tool_objectfs_object_tags', ['objectid' => $objectid]); + + // Store new records. + $recordstostore = []; + foreach ($tags as $key => $value) { + $recordstostore[] = [ + 'objectid' => $objectid, + 'tagkey' => $key, + 'tagvalue' => $value, + ]; + } + $DB->insert_records('tool_objectfs_object_tags', $recordstostore); + } + + /** + * Returns objects that are candidates for tag syncing. + * @param int $limit max number of records to return + * @return array array of contenthashes, which need tags calculated and synced. + */ + public static function get_objects_needing_sync(int $limit) { + global $DB; + + // Find object records where the status is NEEDS_SYNC and is replicated. + [$insql, $inparams] = $DB->get_in_or_equal([ + OBJECT_LOCATION_DUPLICATED, OBJECT_LOCATION_EXTERNAL, OBJECT_LOCATION_ORPHANED], SQL_PARAMS_NAMED); + $inparams['syncstatus'] = self::SYNC_STATUS_NEEDS_SYNC; + $records = $DB->get_records_select('tool_objectfs_objects', 'tagsyncstatus = :syncstatus AND location ' . $insql, + $inparams, '', 'contenthash', 0, $limit); + return array_column($records, 'contenthash'); + } + + /** + * Marks a given object as the given status. + * @param string $contenthash + * @param int $status one of SYNC_STATUS_* constants + * @param int $tagpushedtime if tags were actually sent to the external store, + * this should be the time that happened, or zero if not. + */ + public static function mark_object_tag_sync_status(string $contenthash, int $status, int $tagpushedtime = 0) { + global $DB; + if (!in_array($status, self::SYNC_STATUSES)) { + throw new coding_exception("Invalid object tag sync status " . $status); + } + + $timeupdate = !empty($tagpushedtime) ? ',tagslastpushed = :time' : ''; + $params = [ + 'status' => $status, + 'contenthash' => $contenthash, + 'time' => $tagpushedtime, + ]; + + // Need raw execute since update_records requires an id column, but we use contenthash instead. + $DB->execute("UPDATE {tool_objectfs_objects} + SET tagsyncstatus = :status + {$timeupdate} + WHERE contenthash = :contenthash", + $params); + } + + /** + * Returns a simple list of all the sources and their descriptions. + * @return string html string + */ + public static function get_tag_source_summary_html(): string { + $sources = self::get_defined_tag_sources(); + $table = new html_table(); + $table->head = [ + get_string('table:tagsource', 'tool_objectfs'), + get_string('table:tagsourcemeaning', 'tool_objectfs'), + ]; + + foreach ($sources as $source) { + $table->data[$source->get_identifier()] = [$source->get_identifier(), $source->get_description()]; + } + return html_writer::table($table); + } + + /** + * If the current env is allowed to overwrite tags on objects that already have tags. + * @return bool + */ + public static function can_overwrite_object_tags(): bool { + return (bool) get_config('tool_objectfs', 'overwriteobjecttags'); + } + + /** + * Get the string for a given tag sync status + * @param int $tagsyncstatus one of SYNC_STATUS_* + * @return string + */ + public static function get_sync_status_string(int $tagsyncstatus): string { + $strmap = [ + self::SYNC_STATUS_ERROR => 'error', + self::SYNC_STATUS_NEEDS_SYNC => 'needssync', + self::SYNC_STATUS_COMPLETE => 'notrequired', + ]; + + if (!array_key_exists($tagsyncstatus, $strmap)) { + throw new coding_exception('No status string is mapped for status: ' . $tagsyncstatus); + } + + return get_string('tagsyncstatus:' . $strmap[$tagsyncstatus], 'tool_objectfs'); + } + + /** + * Returns a summary of the object tag sync statuses. + * Note on larger sites, this can be quite computationally difficult and should be used carefully. + * @return array + */ + public static function get_tag_sync_status_summary(): array { + global $DB; + return $DB->get_records_sql("SELECT tagsyncstatus, COUNT(tagsyncstatus) as statuscount + FROM {tool_objectfs_objects} + GROUP BY tagsyncstatus"); + } + + /** + * This is a lightweight check to just check if any objects are reporting tag sync errors. + * @return bool + */ + public static function tag_sync_errors_exist(): bool { + global $DB; + return $DB->record_exists('tool_objectfs_objects', ['tagsyncstatus' => self::SYNC_STATUS_ERROR]); + } +} diff --git a/classes/local/tag/tag_source.php b/classes/local/tag/tag_source.php new file mode 100644 index 00000000..a915c82a --- /dev/null +++ b/classes/local/tag/tag_source.php @@ -0,0 +1,50 @@ +. + +namespace tool_objectfs\local\tag; + +/** + * Tag source interface + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +interface tag_source { + /** + * Returns an unchanging identifier for this source. + * Must never change, otherwise it will lose connection with the tags replicated to objects. + * If it ever must change, a migration step must be completed to trigger all objects to recalculate their tags. + * Must not exceed 128 chars. + * @return string + */ + public static function get_identifier(): string; + + /** + * Description for source displayed in the admin settings. + * @return string + */ + public static function get_description(): string; + + /** + * Returns the value of this tag for the file with the given content hash. + * This must be deterministic, and should never exceed 128 chars. + * @param string $contenthash + * @return string + */ + public function get_value_for_contenthash(string $contenthash): ?string; +} diff --git a/classes/local/tag_sync_count_result.php b/classes/local/tag_sync_count_result.php new file mode 100644 index 00000000..9ce068b1 --- /dev/null +++ b/classes/local/tag_sync_count_result.php @@ -0,0 +1,55 @@ +. + +namespace tool_objectfs\local; + +use core\check\result; +use html_table; +use html_writer; +use tool_objectfs\local\tag\tag_manager; + +/** + * Tagging sync count result + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class tag_sync_count_result extends result { + /** + * Returns the details, which is a table that displays the count for each object status possibility. + * On larger sites, this can take several seconds to execute. + * @return string + */ + public function get_details(): string { + $statuses = tag_manager::get_tag_sync_status_summary(); + $table = new html_table(); + $table->head = [ + get_string('table:status', 'tool_objectfs'), + get_string('table:objectcount', 'tool_objectfs'), + ]; + + foreach (tag_manager::SYNC_STATUSES as $status) { + // If no objects have a status, they won't appear in the SQL above. + // In this case, just show zero (so the use knows it exists, but is zero). + $count = isset($statuses[$status]->statuscount) ? $statuses[$status]->statuscount : 0; + $table->data[$status] = [tag_manager::get_sync_status_string($status), $count]; + } + $table = html_writer::table($table); + return $table; + } +} diff --git a/classes/task/trigger_update_object_tags.php b/classes/task/trigger_update_object_tags.php new file mode 100644 index 00000000..1aeb1b94 --- /dev/null +++ b/classes/task/trigger_update_object_tags.php @@ -0,0 +1,55 @@ +. + +namespace tool_objectfs\task; + +use core\task\manager; +use core\task\scheduled_task; + +/** + * Queues update_object_tags adhoc task periodically, or manually from the frontend. + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class trigger_update_object_tags extends scheduled_task { + /** + * Task name + */ + public function get_name() { + return get_string('task:triggerupdateobjecttags', 'tool_objectfs'); + } + /** + * Execute task + */ + public function execute() { + // Only schedule up to the max amount, less any that are already scheduled. + $alreadyexist = count(manager::get_adhoc_tasks(update_object_tags::class)); + $maxtoschedule = get_config('tool_objectfs', 'maxtaggingtaskstospawn'); + $toschedule = max(0, $maxtoschedule - $alreadyexist); + + for ($i = 0; $i < $toschedule; $i++) { + // Queue adhoc task, nothing else. + $task = new update_object_tags(); + $task->set_custom_data([ + 'iteration' => 1, + ]); + manager::queue_adhoc_task($task); + } + } +} diff --git a/classes/task/update_object_tags.php b/classes/task/update_object_tags.php new file mode 100644 index 00000000..28b5b13c --- /dev/null +++ b/classes/task/update_object_tags.php @@ -0,0 +1,124 @@ +. + +namespace tool_objectfs\task; + +use coding_exception; +use core\task\adhoc_task; +use core\task\manager; +use html_table; +use html_writer; +use moodle_exception; +use tool_objectfs\local\tag\tag_manager; + +/** + * Calculates and updates an objects tags in the external store. + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class update_object_tags extends adhoc_task { + /** + * Returns a status badge depending on the health of the task + * @return string + */ + public function get_status_badge(): string { + $identifier = ''; + $class = ''; + + if ($this->get_fail_delay() > 0) { + $identifier = 'failing'; + $class = 'badge-warning'; + } else if (!is_null($this->get_timestarted()) && $this->get_timestarted() > 0) { + $identifier = 'running'; + $class = 'badge-info'; + } else { + $identifier = 'waiting'; + $class = 'badge-info'; + } + + return html_writer::span(get_string('status:'.$identifier, 'tool_objectfs', $this->get_fail_delay()), 'badge ' . $class); + } + + /** + * Returns iteration count + * @return int + */ + public function get_iteration(): int { + return !empty($this->get_custom_data()->iteration) ? $this->get_custom_data()->iteration : 0; + } + + /** + * Execute task + */ + public function execute() { + if (!tag_manager::is_tagging_enabled_and_supported()) { + // Site admin should know if this migration is running but the fs doesn't support tagging + // (maybe they changed fs mid-run?). + throw new moodle_exception('tagging:migration:notsupported', 'tool_objectfs'); + } + + // Since this adhoc task can requeue itself, ensure there is a fixed limit on the number + // of times this can happen, to avoid any accidental runaways. + $iterationlimit = get_config('tool_objectfs', 'maxtaggingiterations') ?: 0; + $iteration = $this->get_iteration(); + + if (empty($iterationlimit) || empty($iteration) || $iterationlimit < 0 || $iteration < 0) { + // This should never hit here, if it does something is very wrong. + // Throw exception so it causes a retry and alerts. + throw new moodle_exception('tagging:migration:invaliditerations', 'tool_objectfs'); + } + + if ($iteration > $iterationlimit) { + // Generally this means the site has too many objects or not enough configured iterations. + // Regardless it should throw an exception to get the site admins attention. + throw new moodle_exception('tagging:migration:limitreached', 'tool_objectfs', '', $iteration); + } + + $fs = get_file_storage()->get_file_system(); + + // This is checked above in tag_manager::is_tagging_enabled_and_supported, but as a sanity check + // ensure this specific method is defined. + if (!method_exists($fs, "push_object_tags")) { + throw new coding_exception("Filesystem does not define push_object_tags"); + } + + // Get the maximum num of objects to update as configured. + $limit = get_config('tool_objectfs', 'maxtaggingperrun'); + $contenthashes = tag_manager::get_objects_needing_sync($limit); + + if (empty($contenthashes)) { + // This is ok, it means we are done. Exit silently. + mtrace("No more objects found that need tagging, exiting."); + return; + } + + // For each, try to sync their tags. + foreach ($contenthashes as $contenthash) { + $fs->push_object_tags($contenthash); + } + + // Re-queue self to process more in another iteration. + mtrace("Requeing self for another iteration."); + $task = new update_object_tags(); + $task->set_custom_data([ + 'iteration' => $iteration + 1, + ]); + \core\task\manager::queue_adhoc_task($task); + } +} diff --git a/classes/tests/test_client.php b/classes/tests/test_client.php index bda43c5e..12088abc 100644 --- a/classes/tests/test_client.php +++ b/classes/tests/test_client.php @@ -16,6 +16,7 @@ namespace tool_objectfs\tests; +use coding_exception; use tool_objectfs\local\store\object_client_base; /** @@ -37,6 +38,11 @@ class test_client extends object_client_base { */ private $bucketpath; + /** + * @var array in-memory tags used for unit tests + */ + public $tags; + /** * string * @param \stdClass $config @@ -169,5 +175,36 @@ public function get_token_expiry_time(): int { global $CFG; return $CFG->objectfs_phpunit_token_expiry_time; } + + /** + * Sets object tags - uses in-memory store for unit tests + * @param string $contenthash + * @param array $tags + */ + public function set_object_tags(string $contenthash, array $tags) { + global $CFG; + if (!empty($CFG->phpunit_objectfs_simulate_tag_set_error)) { + throw new coding_exception("Simulated tag set error"); + } + $this->tags[$contenthash] = $tags; + } + + /** + * Gets object tags - uses in-memory store for unit tests + * @param string $contenthash + * @return array + */ + public function get_object_tags(string $contenthash): array { + return $this->tags[$contenthash] ?? []; + } + + /** + * Object tagging support, for unit testing + * @return bool + */ + public function supports_object_tagging(): bool { + global $CFG; + return $CFG->phpunit_objectfs_supports_object_tagging; + } } diff --git a/classes/tests/testcase.php b/classes/tests/testcase.php index f3efc8cf..9384e120 100644 --- a/classes/tests/testcase.php +++ b/classes/tests/testcase.php @@ -33,7 +33,6 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ abstract class testcase extends \advanced_testcase { - /** @var test_file_system Filesystem */ public $filesystem; @@ -48,11 +47,35 @@ protected function setUp(): void { global $CFG; $CFG->alternative_file_system_class = '\\tool_objectfs\\tests\\test_file_system'; $CFG->forced_plugin_settings['tool_objectfs']['deleteexternal'] = false; + set_config('taggingenvironment', 'test', 'tool_objectfs'); $this->filesystem = new test_file_system(); $this->logger = new \tool_objectfs\log\null_logger(); + + // Get the file system with reset flag enabled to reset it, + // since it is static and may have been initialised as a filedir system in another test + // instead of the desired objectfs test file system. + get_file_storage(true); + $this->resetAfterTest(true); } + /** + * Enables the test object filesystem and sets the tagging value. + * @param bool $tagging if tagging should be enabled or not. + */ + protected function enable_filesystem_and_set_tagging(bool $tagging) { + global $CFG; + set_config('taggingenabled', $tagging, 'tool_objectfs'); + + // Set supported by fs. + $config = manager::get_objectfs_config(); + $config->taggingenabled = $tagging; + $config->enabletasks = true; + $config->filesystem = '\\tool_objectfs\\tests\\test_file_system'; + manager::set_objectfs_config($config); + $CFG->phpunit_objectfs_supports_object_tagging = $tagging; + } + /** * reset_file_system * @return void diff --git a/db/install.xml b/db/install.xml index 855c9786..0065b1d7 100644 --- a/db/install.xml +++ b/db/install.xml @@ -1,5 +1,5 @@ - @@ -11,6 +11,8 @@ + + @@ -37,7 +39,7 @@ - + @@ -49,5 +51,19 @@ + + + + + + + + + + + + + +
diff --git a/db/tasks.php b/db/tasks.php index ac98e3ff..43e3cfa6 100644 --- a/db/tasks.php +++ b/db/tasks.php @@ -107,5 +107,17 @@ 'dayofweek' => '*', 'month' => '*', ], + [ + 'classname' => 'tool_objectfs\task\trigger_update_object_tags', + 'blocking' => 0, + 'minute' => 'R', + 'hour' => '*', + 'day' => '*', + 'dayofweek' => '*', + 'month' => '*', + // Default disabled - intended to be manually run. + // Also, objectfs tagging support is default off. + 'disabled' => true, + ], ]; diff --git a/db/upgrade.php b/db/upgrade.php index 19c70089..c17b42e1 100644 --- a/db/upgrade.php +++ b/db/upgrade.php @@ -170,5 +170,58 @@ function xmldb_tool_objectfs_upgrade($oldversion) { upgrade_plugin_savepoint(true, 2023013100, 'tool', 'objectfs'); } + + if ($oldversion < 2024110801) { + + // Define table tool_objectfs_object_tags to be created. + $table = new xmldb_table('tool_objectfs_object_tags'); + + // Adding fields to table tool_objectfs_object_tags. + $table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null); + $table->add_field('objectid', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null); + $table->add_field('tagkey', XMLDB_TYPE_CHAR, '32', null, XMLDB_NOTNULL, null, null); + $table->add_field('tagvalue', XMLDB_TYPE_CHAR, '128', null, XMLDB_NOTNULL, null, null); + + // Adding keys to table tool_objectfs_object_tags. + $table->add_key('primary', XMLDB_KEY_PRIMARY, ['id']); + + // Adding indexes to table tool_objectfs_object_tags. + $table->add_index('objecttagkey_idx', XMLDB_INDEX_UNIQUE, ['objectid', 'tagkey']); + + // Conditionally launch create table for tool_objectfs_object_tags. + if (!$dbman->table_exists($table)) { + $dbman->create_table($table); + } + + // Define field tagsyncstatus to be added to tool_objectfs_objects. + $table = new xmldb_table('tool_objectfs_objects'); + $field = new xmldb_field('tagsyncstatus', XMLDB_TYPE_INTEGER, '2', null, XMLDB_NOTNULL, null, '0', 'filesize'); + + // Conditionally launch add field tagsyncstatus. + if (!$dbman->field_exists($table, $field)) { + $dbman->add_field($table, $field); + } + + // Changing precision of field datakey on table tool_objectfs_report_data, + // to (255) to allow for tag key + value pairs to fit in. + $table = new xmldb_table('tool_objectfs_report_data'); + $field = new xmldb_field('datakey', XMLDB_TYPE_CHAR, '255', null, XMLDB_NOTNULL, null, null, 'reporttype'); + + // Launch change of precision for field datakey. + $dbman->change_field_precision($table, $field); + + // Define field tagslastpushed to be added to tool_objectfs_objects. + $table = new xmldb_table('tool_objectfs_objects'); + $field = new xmldb_field('tagslastpushed', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, 0, 'tagsyncstatus'); + + // Conditionally launch add field tagslastpushed. + if (!$dbman->field_exists($table, $field)) { + $dbman->add_field($table, $field); + } + + // Objectfs savepoint reached. + upgrade_plugin_savepoint(true, 2024110801, 'tool', 'objectfs'); + } + return true; } diff --git a/lang/en/tool_objectfs.php b/lang/en/tool_objectfs.php index 7a341a91..cf51606c 100644 --- a/lang/en/tool_objectfs.php +++ b/lang/en/tool_objectfs.php @@ -277,3 +277,62 @@ $string['check:tokenexpiry:expired'] = 'Token expired for {$a->dayssince} days. Expired on {$a->time}'; $string['check:tokenexpiry:na'] = 'Token expiry check not implemented for filesystem, or no token is set'; $string['settings:tokenexpirywarnperiod'] = 'Token expiry warn period'; + +$string['settings:taggingenvironment'] = 'Tagging environment'; +$string['settings:taggingenvironment:desc'] = 'The \'environment\' tag value. Used to distinguish the source of objects when multiple environments share a single bucket.'; +$string['settings:taggingheader'] = 'Tagging settings'; +$string['settings:taggingenabled'] = 'Tagging enabled'; +$string['settings:maxtaggingperrun'] = 'Object tagging adhoc sync maximum objects per run'; +$string['settings:maxtaggingperrun:desc'] = 'The maximum number of objects to sync tags for per tagging sync adhoc task iteration.'; +$string['settings:maxtaggingiterations'] = 'Object tagging adhoc sync maximum number of iterations '; +$string['settings:maxtaggingiterations:desc'] = 'The maximum number of times the tagging sync adhoc task will requeue itself. To avoid accidental infinite runaway.'; +$string['settings:maxtaggingtaskstospawn'] = 'Maximum number of parallel tagging migration tasks'; +$string['settings:maxtaggingtaskstospawn:desc'] = 'Each trigger of the scheduled task `trigger_update_object_tags` will spawn this amount of tasks, minus those that are already running.'; +$string['settings:overrideobjecttags'] = 'Allow object tag override'; +$string['settings:overrideobjecttags:desc'] = 'Allows ObjectFS to overwrite tags on objects that already exist in the external store. If not checked, objectfs will only set tags when the objects "environment" value is empty or is the same as currently defined.'; +$string['settings:tagsources'] = 'Tag sources'; +$string['settings:taggingstatus'] = 'Tagging status'; +$string['settings:taggingstatuscounts'] = 'Tag sync status overview'; +$string['settings:taggingmigrationstatus'] = 'Tagging adhoc migration progress'; +$string['settings:tagging:help'] = 'Object tagging allows extra metadata to be attached to objects in the external store. Please read TAGGING.md in the plugin Github repository for detailed setup and considerations. This is currently only supported by the S3 external client.'; + +$string['checktagging_status'] = 'Object tagging'; +$string['checktagging_sync_status'] = 'Object tagging sync status'; +$string['checktagging_migration_status'] = 'Object tagging migration status'; + +$string['check:tagging:ok'] = 'Object tagging ok'; +$string['check:tagging:syncerror'] = 'Objects have tag sync errors'; +$string['check:tagging:syncok'] = 'No objects reporting sync errors'; +$string['check:tagging:migrationerror'] = 'Object tagging migration task(s) have faildelay > 0'; +$string['check:tagging:migrationok'] = 'Object tagging migration tasks OK'; +$string['check:tagging:na'] = 'Tagging not enabled or is not supported by file system'; +$string['check:tagging:error'] = 'Error trying to tag object'; + +$string['tagsource:environment'] = 'Environment defined by the "taggingenvironment" setting, currently: "{$a}".'; +$string['tagsource:environment:toolong'] = 'The value defined in objectfs_environment_name is too long. It must be < 128 chars'; +$string['tagsource:location'] = 'Location of file, either "orphan" or "active".'; + +$string['task:triggerupdateobjecttags'] = 'Queue adhoc task to update object tags'; + +$string['tagsyncstatus:error'] = 'Errored'; +$string['tagsyncstatus:notrequired'] = 'Not required / synced'; +$string['tagsyncstatus:needssync'] = 'Waiting for sync'; + +$string['tagging:migration:notsupported'] = 'Tagging not enabled or supported by filesystem. Cannot execute tag migration task'; +$string['tagging:migration:invaliditerations'] = 'Invalid iteration number or iteration count'; +$string['tagging:migration:limitreached'] = 'Current iteration {$a} is >= the maximum number of iterations. Please investigate if this is expected and you need to increase the limit, or if there is a problem syncing the tags causing an infinite loop'; +$string['tagging:migration:nothingrunning'] = 'No tagging migration adhoc tasks are currently running'; +$string['tagging:migration:help'] = 'Run the trigger_update_object_tags scheduled task from the frontend or CLI to start a migration task.'; + +$string['table:taskid'] = 'Task ID'; +$string['table:iteration'] = 'Iteration number'; +$string['table:status'] = 'Status'; +$string['table:objectcount'] = 'Object count'; +$string['table:tagsource'] = 'Tag source'; +$string['table:tagsourcemeaning'] = 'Description'; + +$string['status:waiting'] = 'Waiting'; +$string['status:running'] = 'Running'; +$string['status:failing'] = 'Faildelay {$a}'; + +$string['object_status:tag_count'] = 'Object tags'; diff --git a/lib.php b/lib.php index 708063a8..ad6fe853 100644 --- a/lib.php +++ b/lib.php @@ -24,6 +24,7 @@ */ use tool_objectfs\local\object_manipulator\manipulator_builder; +use tool_objectfs\local\tag\tag_manager; define('OBJECTFS_PLUGIN_NAME', 'tool_objectfs'); @@ -103,6 +104,9 @@ function tool_objectfs_pluginfile($course, $cm, context $context, $filearea, arr function tool_objectfs_status_checks() { $checks = [ new tool_objectfs\check\token_expiry(), + new tool_objectfs\check\tagging_status(), + new tool_objectfs\check\tagging_sync_status(), + new tool_objectfs\check\tagging_migration_status(), ]; if (get_config('tool_objectfs', 'proxyrangerequests')) { diff --git a/object_status.php b/object_status.php index 7161aa9a..84ece33c 100644 --- a/object_status.php +++ b/object_status.php @@ -57,6 +57,9 @@ echo $OUTPUT->box_start(); $table = new object_status_history_table($reporttype, $reportid); $table->baseurl = $pageurl; + + $heading = get_string('object_status:' . $reporttype, 'tool_objectfs'); + echo $OUTPUT->heading($heading, 2); $table->out(0, false); echo $OUTPUT->box_end(); } diff --git a/settings.php b/settings.php index 5ed74fef..dbad83e5 100644 --- a/settings.php +++ b/settings.php @@ -24,6 +24,9 @@ */ use tool_objectfs\check\token_expiry; +use tool_objectfs\check\tagging_migration_status; +use tool_objectfs\check\tagging_sync_status; +use tool_objectfs\local\tag\tag_manager; defined('MOODLE_INTERNAL') || die(); @@ -131,7 +134,6 @@ $settings->add(new admin_setting_configduration('tool_objectfs/consistencydelay', new lang_string('settings:consistencydelay', 'tool_objectfs'), '', 10 * MINSECS, MINSECS)); - $settings->add(new admin_setting_heading('tool_objectfs/storagefilesystemselection', new lang_string('settings:clientselection:header', 'tool_objectfs'), '')); @@ -267,4 +269,72 @@ $settings->add(new admin_setting_configcheckbox('tool_objectfs/preferexternal', new lang_string('settings:preferexternal', 'tool_objectfs'), '', '')); + + // Tagging settings. + $settings->add(new admin_setting_heading('tool_objectfs/taggingsettings', + new lang_string('settings:taggingheader', 'tool_objectfs'), + get_string('settings:tagging:help', 'tool_objectfs') + )); + + $settings->add(new admin_setting_configcheckbox('tool_objectfs/taggingenabled', + new lang_string('settings:taggingenabled', 'tool_objectfs'), '', 0)); + + $settings->add(new admin_setting_configtext('tool_objectfs/taggingenvironment', + new lang_string('settings:taggingenvironment', 'tool_objectfs'), + get_string('settings:taggingenvironment:desc', 'tool_objectfs'), + '', + PARAM_TEXT + )); + + $settings->add(new admin_setting_description('tool_objectfs/tagsources', + new lang_string('settings:tagsources', 'tool_objectfs'), + tag_manager::get_tag_source_summary_html() + )); + + $settings->add(new admin_setting_configtext('tool_objectfs/maxtaggingperrun', + new lang_string('settings:maxtaggingperrun', 'tool_objectfs'), + get_string('settings:maxtaggingperrun:desc', 'tool_objectfs'), + 1000, + PARAM_INT + )); + + $settings->add(new admin_setting_configtext('tool_objectfs/maxtaggingiterations', + new lang_string('settings:maxtaggingiterations', 'tool_objectfs'), + get_string('settings:maxtaggingiterations:desc', 'tool_objectfs'), + 10000, + PARAM_INT + )); + + $settings->add(new admin_setting_configtext('tool_objectfs/maxtaggingtaskstospawn', + new lang_string('settings:maxtaggingtaskstospawn', 'tool_objectfs'), + get_string('settings:maxtaggingtaskstospawn:desc', 'tool_objectfs'), + 1, + PARAM_INT + )); + + $settings->add(new admin_setting_configcheckbox('tool_objectfs/overwriteobjecttags', + new lang_string('settings:overrideobjecttags', 'tool_objectfs'), + get_string('settings:overrideobjecttags:desc', 'tool_objectfs'), + 1 + )); + + // Tagging status. + $settings->add(new admin_setting_heading('tool_objectfs/taggingstatus', + new lang_string('settings:taggingstatus', 'tool_objectfs'), '')); + + // Only in 4.4+. + if (class_exists('admin_setting_check')) { + $settings->add(new admin_setting_check('tool_objectfs/check_taggingsyncstatus', new tagging_sync_status(), true)); + $settings->add(new admin_setting_check('tool_objectfs/check_taggingmigrationstatus', new tagging_migration_status(), true)); + } else { + // Fallback to links instead. + $settings->add(new admin_setting_description('taggingstatuslink', '', html_writer::link( + new moodle_url('/report/status/index.php', ['detail' => 'tool_objectfs_tagging_sync_status']), + get_string('settings:taggingstatus', 'tool_objectfs') + ))); + $settings->add(new admin_setting_description('taggingmigrationstatuslink', '', html_writer::link( + new moodle_url('/report/status/index.php', ['detail' => 'tool_objectfs_tagging_migration_status']), + get_string('settings:taggingmigrationstatus', 'tool_objectfs') + ))); + } } diff --git a/tests/check/tagging_migration_status_test.php b/tests/check/tagging_migration_status_test.php new file mode 100644 index 00000000..3b9763c7 --- /dev/null +++ b/tests/check/tagging_migration_status_test.php @@ -0,0 +1,73 @@ +. + +namespace tool_objectfs\check; + +use core\check\result; +use core\task\manager; +use tool_objectfs\check\tagging_migration_status; +use tool_objectfs\task\update_object_tags; +use tool_objectfs\tests\testcase; + +/** + * Tagging migration status check tests + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @covers \tool_objectfs\check\tagging_migration_status + */ +class tagging_migration_status_test extends testcase { + /** + * Tests scenario that returns N/A + */ + public function test_get_result_na() { + // Regardless if this is disabled, the check should still return a non n/a status. + $this->enable_filesystem_and_set_tagging(false); + $check = new tagging_migration_status(); + $this->assertEquals(result::NA, $check->get_result()->get_status()); + } + + /* + * Test scenario that returns WARNING + */ + public function test_get_result_warning() { + // Regardless if this is disabled, the check should still return a non n/a status. + $this->enable_filesystem_and_set_tagging(false); + + $task = new update_object_tags(); + $task->set_fail_delay(64); + manager::queue_adhoc_task($task); + + $check = new tagging_migration_status(); + $this->assertEquals(result::WARNING, $check->get_result()->get_status()); + } + + /* + * Test scenario that returns OK + */ + public function test_get_result_ok() { + // Regardless if this is disabled, the check should still return a non n/a status. + $this->enable_filesystem_and_set_tagging(false); + + $task = new update_object_tags(); + manager::queue_adhoc_task($task); + + $check = new tagging_migration_status(); + $this->assertEquals(result::OK, $check->get_result()->get_status()); + } +} diff --git a/tests/check/tagging_sync_status_test.php b/tests/check/tagging_sync_status_test.php new file mode 100644 index 00000000..eba545cf --- /dev/null +++ b/tests/check/tagging_sync_status_test.php @@ -0,0 +1,68 @@ +. + +namespace tool_objectfs\check; + +use core\check\result; +use tool_objectfs\check\tagging_sync_status; +use tool_objectfs\local\tag\tag_manager; +use tool_objectfs\tests\testcase; + +/** + * Tagging sync status check tests + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @covers \tool_objectfs\check\tagging_sync_status + */ +class tagging_sync_status_test extends testcase { + /** + * Tests scenario that returns N/A + */ + public function test_get_result_na() { + // Not enabled by default, should return N/A. + $check = new tagging_sync_status(); + $this->assertEquals(result::NA, $check->get_result()->get_status()); + } + + /** + * Test scenario that returns OK + */ + public function test_get_result_ok() { + $this->enable_filesystem_and_set_tagging(true); + $object = $this->create_remote_object(); + tag_manager::mark_object_tag_sync_status($object->contenthash, tag_manager::SYNC_STATUS_COMPLETE); + + // All objects OK, should return ok. + $check = new tagging_sync_status(); + $this->assertEquals(result::OK, $check->get_result()->get_status()); + } + + /** + * Tests scenario that returns WARNING + */ + public function test_get_result_warning() { + $this->enable_filesystem_and_set_tagging(true); + $object = $this->create_remote_object(); + tag_manager::mark_object_tag_sync_status($object->contenthash, tag_manager::SYNC_STATUS_ERROR); + + // An object has error, should return warning. + $check = new tagging_sync_status(); + $this->assertEquals(result::WARNING, $check->get_result()->get_status()); + } +} diff --git a/tests/local/report/object_status_test.php b/tests/local/report/object_status_test.php index bbc895d4..a06caa53 100644 --- a/tests/local/report/object_status_test.php +++ b/tests/local/report/object_status_test.php @@ -66,7 +66,7 @@ public function test_generate_status_report_historic() { public function test_get_report_types() { $reporttypes = objectfs_report::get_report_types(); $this->assertEquals('array', gettype($reporttypes)); - $this->assertEquals(3, count($reporttypes)); + $this->assertEquals(4, count($reporttypes)); } /** diff --git a/tests/local/tagging_test.php b/tests/local/tagging_test.php new file mode 100644 index 00000000..9be6839d --- /dev/null +++ b/tests/local/tagging_test.php @@ -0,0 +1,442 @@ +. + +namespace tool_objectfs\local; + +use coding_exception; +use moodle_exception; +use Throwable; +use tool_objectfs\local\manager; +use tool_objectfs\local\tag\environment_source; +use tool_objectfs\local\tag\tag_manager; +use tool_objectfs\local\tag\tag_source; +use tool_objectfs\tests\test_client; +use tool_objectfs\tests\testcase; + +/** + * Tests tagging + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class tagging_test extends testcase { + /** + * Tests get_defined_tag_sources + * @covers \tool_objectfs\local\tag_manager::get_defined_tag_sources + */ + public function test_get_defined_tag_sources() { + $sources = tag_manager::get_defined_tag_sources(); + $this->assertIsArray($sources); + + // Both AWS and Azure limit 10 tags per object, so ensure never more than 10 sources defined. + $this->assertLessThanOrEqual(10, count($sources)); + } + + /** + * Provides values to various tag source tests + * @return array + */ + public static function tag_source_provider(): array { + $sources = tag_manager::get_defined_tag_sources(); + $tests = []; + + foreach ($sources as $source) { + $tests[$source->get_identifier()] = [ + 'source' => $source, + ]; + } + + return $tests; + } + + /** + * Tests the source identifier + * @param tag_source $source + * @dataProvider tag_source_provider + * @covers \tool_objectfs\local\tag_source::get_identifier + */ + public function test_tag_sources_identifier(tag_source $source) { + $count = strlen($source->get_identifier()); + + // Ensure < 32 chars, the max length as defined in our docs. + $this->assertLessThan(32, $count); + $this->assertGreaterThan(0, $count); + } + + /** + * Tests the source value + * @param tag_source $source + * @dataProvider tag_source_provider + * @covers \tool_objectfs\local\tag_source::get_value_for_contenthash + */ + public function test_tag_sources_value(tag_source $source) { + $file = $this->create_duplicated_object('tag source value test ' . $source->get_identifier()); + $value = $source->get_value_for_contenthash($file->contenthash); + + // Null value - allowed, but means we cannot test. + if (is_null($value)) { + return; + } + + $count = strlen($value); + + // Ensure < 128 chars, the max length as defined in our docs. + $this->assertLessThan(128, $count); + $this->assertGreaterThan(0, $count); + } + + /** + * Provides values to test_is_tagging_enabled_and_supported + * @return array + */ + public static function is_tagging_enabled_and_supported_provider(): array { + return [ + 'neither config nor fs supports' => [ + 'enabledinconfig' => false, + 'supportedbyfs' => false, + 'expected' => false, + ], + 'enabled in config but fs does not support' => [ + 'enabledinconfig' => true, + 'supportedbyfs' => false, + 'expected' => false, + ], + 'enabled in config and fs does support' => [ + 'enabledinconfig' => true, + 'supportedbyfs' => true, + 'expected' => true, + ], + ]; + } + + /** + * Tests is_tagging_enabled_and_supported + * @param bool $enabledinconfig if tagging feature is turned on + * @param bool $supportedbyfs if the filesystem supports tagging + * @param bool $expected expected return result + * @dataProvider is_tagging_enabled_and_supported_provider + * @covers \tool_objectfs\local\tag_manager::is_tagging_enabled_and_supported + */ + public function test_is_tagging_enabled_and_supported(bool $enabledinconfig, bool $supportedbyfs, bool $expected) { + global $CFG; + // Set config. + set_config('taggingenabled', $enabledinconfig, 'tool_objectfs'); + + // Set supported by fs. + $config = manager::get_objectfs_config(); + $config->taggingenabled = $enabledinconfig; + $config->enabletasks = true; + $config->filesystem = '\\tool_objectfs\\tests\\test_file_system'; + manager::set_objectfs_config($config); + $CFG->phpunit_objectfs_supports_object_tagging = $supportedbyfs; + + $this->assertEquals($expected, tag_manager::is_tagging_enabled_and_supported()); + } + + /** + * Tests gather_object_tags_for_upload + * @covers \tool_objectfs\local\tag_manager::gather_object_tags_for_upload + */ + public function test_gather_object_tags_for_upload() { + $object = $this->create_duplicated_object('gather tags for upload test'); + $tags = tag_manager::gather_object_tags_for_upload($object->contenthash); + + $this->assertArrayHasKey('environment', $tags); + $this->assertEquals('test', $tags['environment']); + $this->assertArrayHasKey('location', $tags); + $this->assertEquals('active', $tags['location']); + } + + /** + * Tests gather_object_tags_for_upload when orphaned + * @covers \tool_objectfs\local\tag_manager::gather_object_tags_for_upload + */ + public function test_gather_object_tags_for_upload_orphaned() { + global $DB; + $object = $this->create_duplicated_object('gather tags for upload test'); + + // Change the object record to be orphaned. + $DB->update_record('tool_objectfs_objects', ['id' => $object->id, 'location' => OBJECT_LOCATION_ORPHANED]); + + $tags = tag_manager::gather_object_tags_for_upload($object->contenthash); + + $this->assertArrayHasKey('environment', $tags); + $this->assertEquals('test', $tags['environment']); + $this->assertArrayHasKey('location', $tags); + $this->assertEquals('orphan', $tags['location']); + } + + /** + * Tests store_tags_locally + * @covers \tool_objectfs\local\tag_manager::store_tags_locally + */ + public function test_store_tags_locally() { + global $DB; + + $tags = [ + 'test1' => 'abc', + 'test2' => 'xyz', + ]; + $object = $this->create_remote_object(); + + // Ensure no tags for hash intially. + $this->assertEmpty($DB->get_records('tool_objectfs_object_tags', ['objectid' => $object->id])); + + // Store. + tag_manager::store_tags_locally($object->contenthash, $tags); + + // Confirm they are stored. + $queriedtags = $DB->get_records('tool_objectfs_object_tags', ['objectid' => $object->id]); + $this->assertCount(2, $queriedtags); + } + + /** + * Provides values to test_get_objects_needing_sync + * @return array + */ + public static function get_objects_needing_sync_provider(): array { + return [ + 'duplicated, needs sync' => [ + 'location' => OBJECT_LOCATION_DUPLICATED, + 'status' => tag_manager::SYNC_STATUS_NEEDS_SYNC, + 'expectedneedssync' => true, + ], + 'remote, needs sync' => [ + 'location' => OBJECT_LOCATION_EXTERNAL, + 'status' => tag_manager::SYNC_STATUS_NEEDS_SYNC, + 'expectedneedssync' => true, + ], + 'local, needs sync' => [ + 'location' => OBJECT_LOCATION_LOCAL, + 'status' => tag_manager::SYNC_STATUS_NEEDS_SYNC, + 'expectedneedssync' => false, + ], + 'duplicated, does not need sync' => [ + 'location' => OBJECT_LOCATION_DUPLICATED, + 'status' => tag_manager::SYNC_STATUS_COMPLETE, + 'expectedneedssync' => false, + ], + 'local, does not need sync' => [ + 'location' => OBJECT_LOCATION_LOCAL, + 'status' => tag_manager::SYNC_STATUS_COMPLETE, + 'expectedneedssync' => false, + ], + 'duplicated, sync error' => [ + 'location' => OBJECT_LOCATION_DUPLICATED, + 'status' => tag_manager::SYNC_STATUS_ERROR, + 'expectedneedssync' => false, + ], + 'local, sync error' => [ + 'location' => OBJECT_LOCATION_LOCAL, + 'status' => tag_manager::SYNC_STATUS_ERROR, + 'expectedneedssync' => false, + ], + ]; + } + + /** + * Tests get_objects_needing_sync + * @param int $location object location + * @param int $syncstatus sync status to set on object record + * @param bool $expectedneedssync if the object should be included in the return of the function + * @dataProvider get_objects_needing_sync_provider + * @covers \tool_objectfs\local\tag_manager::get_objects_needing_sync + */ + public function test_get_objects_needing_sync(int $location, int $syncstatus, bool $expectedneedssync) { + global $DB; + + // Create the test object at the required location. + switch ($location) { + case OBJECT_LOCATION_DUPLICATED: + $object = $this->create_duplicated_object('tagging test object duplicated'); + break; + case OBJECT_LOCATION_LOCAL: + $object = $this->create_local_object('tagging test object local'); + break; + case OBJECT_LOCATION_EXTERNAL: + $object = $this->create_remote_object('tagging test object remote'); + break; + default: + throw new coding_exception("Object location not handled in test"); + } + + // Set the sync status. + $DB->set_field('tool_objectfs_objects', 'tagsyncstatus', $syncstatus, ['id' => $object->id]); + + // Check if it is included in the list. + $needssync = tag_manager::get_objects_needing_sync(1); + + if ($expectedneedssync) { + $this->assertContains($object->contenthash, $needssync); + } else { + $this->assertNotContains($object->contenthash, $needssync); + } + } + + /** + * Tests the limit input to get_objects_needing_sync + * @covers \tool_objectfs\local\tag_manager::get_objects_needing_sync + */ + public function test_get_objects_needing_sync_limit() { + global $DB; + + // Create two duplicated objects needing sync. + $object = $this->create_duplicated_object('sync limit test duplicated'); + $DB->set_field('tool_objectfs_objects', 'tagsyncstatus', tag_manager::SYNC_STATUS_NEEDS_SYNC, ['id' => $object->id]); + $object = $this->create_remote_object('sync limit test remote'); + $DB->set_field('tool_objectfs_objects', 'tagsyncstatus', tag_manager::SYNC_STATUS_NEEDS_SYNC, ['id' => $object->id]); + + // Ensure a limit of 2 returns 2, and limit of 1 returns 1. + $this->assertCount(2, tag_manager::get_objects_needing_sync(2)); + $this->assertCount(1, tag_manager::get_objects_needing_sync(1)); + } + + /** + * Test get_tag_source_summary_html + * @covers \tool_objectfs\local\tag_manager::get_tag_source_summary_html + */ + public function test_get_tag_source_summary_html() { + // Quick test just to ensure it generates and nothing explodes. + $html = tag_manager::get_tag_source_summary_html(); + $this->assertIsString($html); + } + + /** + * Tests when fails to sync object tags, that the sync status is updated to SYNC_STATUS_ERROR. + * @covers \tool_objectfs\local\tag_manager + */ + public function test_object_tag_sync_error() { + global $CFG, $DB; + + // Integration clients won't simulate errors, so we can't test this functionality. + if (!$this->filesystem->externalclient instanceof test_client) { + $this->markTestSkipped("Cannot test sync error during integration testing"); + return; + } + + // Setup FS for tagging. + $config = manager::get_objectfs_config(); + $config->taggingenabled = true; + $config->enabletasks = true; + $config->filesystem = '\\tool_objectfs\\tests\\test_file_system'; + manager::set_objectfs_config($config); + + $CFG->phpunit_objectfs_supports_object_tagging = true; + $this->assertTrue(tag_manager::is_tagging_enabled_and_supported()); + + // Create a good duplicated object. + $object = $this->create_duplicated_object('sync limit test duplicated'); + $status = $DB->get_field('tool_objectfs_objects', 'tagsyncstatus', ['id' => $object->id]); + $this->assertEquals(tag_manager::SYNC_STATUS_COMPLETE, $status); + + // Now try push tags, but trigger a simulated tag set error. + $CFG->phpunit_objectfs_simulate_tag_set_error = true; + $didthrow = false; + try { + $this->filesystem->push_object_tags($object->contenthash); + } catch (Throwable $e) { + $didthrow = true; + } + $this->assertTrue($didthrow); + + // Ensure tag sync status set to error. + $status = $DB->get_field('tool_objectfs_objects', 'tagsyncstatus', ['id' => $object->id]); + $this->assertEquals(tag_manager::SYNC_STATUS_ERROR, $status); + } + + /** + * Tests tag_manger::get_tag_sync_status_summary + * @covers \tool_objectfs\local\tag_manager::get_tag_sync_status_summary + */ + public function test_get_tag_sync_status_summary() { + // Ensure clean slate before test starts. + global $DB; + $DB->delete_records('tool_objectfs_objects'); + + // Create an object with each status. + $object1 = $this->create_local_object('test1'); + $object2 = $this->create_local_object('test2'); + $object3 = $this->create_local_object('test3'); + + // Delete the unit test object that is automatically created, it has a filesize of zero. + $DB->delete_records('tool_objectfs_objects', ['filesize' => 0]); + + tag_manager::mark_object_tag_sync_status($object1->contenthash, tag_manager::SYNC_STATUS_COMPLETE); + tag_manager::mark_object_tag_sync_status($object2->contenthash, tag_manager::SYNC_STATUS_ERROR); + tag_manager::mark_object_tag_sync_status($object3->contenthash, tag_manager::SYNC_STATUS_NEEDS_SYNC); + + // Ensure correctly counted. + $statuses = tag_manager::get_tag_sync_status_summary(); + $this->assertEquals(1, $statuses[tag_manager::SYNC_STATUS_COMPLETE]->statuscount); + $this->assertEquals(1, $statuses[tag_manager::SYNC_STATUS_ERROR]->statuscount); + $this->assertEquals(1, $statuses[tag_manager::SYNC_STATUS_NEEDS_SYNC]->statuscount); + } + + /** + * Provides sync statuses to tests + * @return array + */ + public static function sync_status_provider(): array { + $tests = []; + foreach (tag_manager::SYNC_STATUSES as $status) { + $tests[$status] = [ + 'status' => $status, + ]; + } + return $tests; + } + + /** + * Tests get_sync_status_string + * @param int $status + * @dataProvider sync_status_provider + * @covers \tool_objectfs\local\tag_manager::get_sync_status_string + */ + public function test_get_sync_status_string(int $status) { + $string = tag_manager::get_sync_status_string($status); + // Cheap check to ensure placeholder string not returned. + $this->assertStringNotContainsString('[', $string); + } + + /** + * Tests get_sync_status_string when an invalid status is provided + * @covers \tool_objectfs\local\tag_manager::get_sync_status_string + */ + public function test_get_sync_status_string_does_not_exist() { + $this->expectException(coding_exception::class); + $this->expectExceptionMessage('No status string is mapped for status: 5'); + tag_manager::get_sync_status_string(5); + } + + /** + * Tests the length of the defined tag source is checked correctly + * @covers \tool_objectfs\local\environment_source + */ + public function test_environment_source_too_long() { + global $CFG; + set_config('taggingenvironment', 'This is a really long string. + It needs to be long because it needs to be more than 128 chars for the test to trigger an exception.', + 'tool_objectfs'); + + $source = new environment_source(); + + $this->expectException(moodle_exception::class); + $this->expectExceptionMessage(get_string('tagsource:environment:toolong', 'tool_objectfs')); + $source->get_value_for_contenthash('test'); + } +} diff --git a/tests/object_file_system_test.php b/tests/object_file_system_test.php index 0abe42a7..405be19e 100644 --- a/tests/object_file_system_test.php +++ b/tests/object_file_system_test.php @@ -16,8 +16,10 @@ namespace tool_objectfs; +use coding_exception; use tool_objectfs\local\store\object_file_system; use tool_objectfs\local\manager; +use tool_objectfs\local\tag\tag_manager; use tool_objectfs\tests\test_file_system; /** @@ -1038,4 +1040,148 @@ public function test_add_file_from_string_update_object_fail() { $this->assertEquals(\core_text::strlen($content), $result[1]); $this->assertTrue($result[2]); } + + /** + * Test syncing tags throws exception when client does not support tagging. + */ + public function test_push_object_tags_not_supported() { + global $CFG; + $CFG->phpunit_objectfs_supports_object_tagging = false; + $this->expectException(coding_exception::class); + $this->expectExceptionMessage('Cannot sync tags, external client does not support tagging'); + $this->filesystem->push_object_tags('123'); + } + + /** + * Tests syncing object tags where the file is not replicated. + */ + public function test_push_object_tags_object_not_replicated() { + global $CFG, $DB; + $CFG->phpunit_objectfs_supports_object_tagging = true; + + // Create object - not replicated to 'external' store yet. + $object = $this->create_local_object('test syncing local'); + + // Sync, this should do nothing but change sync status - cannot sync object tags + // where the object is not replicated. + $this->filesystem->push_object_tags($object->contenthash); + $object = $DB->get_record('tool_objectfs_objects', ['contenthash' => $object->contenthash]); + $this->assertEquals($object->tagsyncstatus, tag_manager::SYNC_STATUS_COMPLETE); + } + + /** + * Provides values to push_object_tags_replicated + * @return array + */ + public static function push_object_tags_replicated_provider(): array { + return [ + // Can override, doesn't matter if envs are different. + 'can override - different env' => [ + 'object env' => 'prod', + 'push env' => 'staging', + 'can override' => true, + 'expected override' => true, + ], + 'can override - same env' => [ + 'object env' => 'prod', + 'push env' => 'prod', + 'can override' => true, + 'expected override' => true, + ], + 'can override - empty env' => [ + 'object env' => '', + 'push env' => 'prod', + 'can override' => true, + 'expected override' => true, + ], + // Cannot override, env must match or be empty. + 'cannot override - same env' => [ + 'object env' => 'prod', + 'push env' => 'prod', + 'can override' => false, + 'expected override' => true, + ], + 'cannot override - different env' => [ + 'object env' => 'prod', + 'push env' => 'staging', + 'can override' => false, + 'expected override' => false, + ], + 'cannot override - env is empty' => [ + 'object env' => '', + 'push env' => 'staging', + 'can override' => false, + 'expected override' => true, + ], + ]; + } + + /** + * Tests push_object_tags when the object is replicated. + * Tests rules around overriding are correctly applied. + * + * @param string $objectenv the env to set when 'uploading' the object + * @param string $pushenv the env to set when trying to push new tags + * @param bool $canoverride if filesystem should be able to overwrite existing objects + * @param bool $expectedoverride if it was expected that the tags were overwritten. + * @dataProvider push_object_tags_replicated_provider + */ + public function test_push_object_tags_replicated(string $objectenv, string $pushenv, bool $canoverride, + bool $expectedoverride) { + global $CFG, $DB; + $CFG->phpunit_objectfs_supports_object_tagging = true; + set_config('taggingenvironment', $objectenv, 'tool_objectfs'); + + set_config('overwriteobjecttags', $canoverride, 'tool_objectfs'); + $this->assertEquals($canoverride, tag_manager::can_overwrite_object_tags()); + + $object = $this->create_duplicated_object('test syncing replicated'); + $testtags = tag_manager::gather_object_tags_for_upload($object->contenthash); + + // Fake set the tags in the external store. + $this->filesystem->get_external_client()->tags[$object->contenthash] = $testtags; + + // Ensure tags are set 'externally'. + $tags = $this->filesystem->get_external_client()->get_object_tags($object->contenthash); + $this->assertCount(count($testtags), $tags); + + // But tags will not be stored locally (yet). + $localtags = $DB->get_records('tool_objectfs_object_tags', ['objectid' => $object->id]); + $this->assertCount(0, $localtags); + + set_config('taggingenvironment', $pushenv, 'tool_objectfs'); + + // Sync the file. + $this->filesystem->push_object_tags($object->contenthash); + + // Tags should now be replicated locally. + $localtags = $DB->get_records('tool_objectfs_object_tags', ['objectid' => $object->id]); + $externaltags = $this->filesystem->get_external_client()->get_object_tags($object->contenthash); + $time = $DB->get_field('tool_objectfs_objects', 'tagslastpushed', ['id' => $object->id]); + + if ($expectedoverride) { + // If can override, we expect it to be overwritten by the tags defined in the sources. + $expectednum = count(tag_manager::get_defined_tag_sources()); + $this->assertCount($expectednum, $localtags); + + // Also expect the external store to be updated. + $this->assertCount($expectednum, $externaltags); + + // Tag push time should be set, since it actually pushed the tags. + $this->assertNotEquals(0, $time); + } else { + // If cannot overwrite, no tags should be synced. + $this->assertCount(0, $localtags); + + // External store should not be changed. + $this->assertCount(count($testtags), $externaltags); + + // The tag last push time should remain unchanged, since it didn't actually push any tags. + $this->assertEquals(0, $time); + } + + // Ensure status changed to not needing sync. + $object = $DB->get_record('tool_objectfs_objects', ['contenthash' => $object->contenthash]); + $this->assertEquals($object->tagsyncstatus, tag_manager::SYNC_STATUS_COMPLETE); + } } diff --git a/tests/task/populate_objects_filesize_test.php b/tests/task/populate_objects_filesize_test.php index fdb9d62d..6986ba6d 100644 --- a/tests/task/populate_objects_filesize_test.php +++ b/tests/task/populate_objects_filesize_test.php @@ -27,13 +27,6 @@ */ class populate_objects_filesize_test extends \tool_objectfs\tests\testcase { - /** - * This method runs before every test. - */ - public function setUp(): void { - $this->resetAfterTest(); - } - /** * Test multiple objects have their filesize updated. */ @@ -214,6 +207,8 @@ public function test_orphaned_objects_are_not_updated() { */ public function test_objects_with_error_are_not_updated() { global $DB; + $numstart = $DB->count_records('tool_objectfs_objects'); + $file1 = $this->create_local_file("Test 1"); $this->create_local_file("Test 2"); $this->create_local_file("Test 3"); @@ -237,7 +232,7 @@ public function test_objects_with_error_are_not_updated() { }); // Test that 4 records have now been updated. - $this->assertCount(5, $objects); - $this->assertCount(4, $updatedobjects); + $this->assertEquals(5, count($objects) - $numstart); + $this->assertEquals(4, count($updatedobjects) - $numstart); } } diff --git a/tests/task/trigger_update_object_tags_test.php b/tests/task/trigger_update_object_tags_test.php new file mode 100644 index 00000000..53413003 --- /dev/null +++ b/tests/task/trigger_update_object_tags_test.php @@ -0,0 +1,50 @@ +. + +namespace tool_objectfs\task; + +use advanced_testcase; +use core\task\manager; + +/** + * Tests trigger_update_object_tags + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class trigger_update_object_tags_test extends advanced_testcase { + /** + * Tests executing scheduled task. + * @covers \tool_objectfs\task\trigger_update_object_tags::execute + */ + public function test_execute() { + $this->resetAfterTest(); + + $task = new trigger_update_object_tags(); + $task->execute(); + + // Ensure it spawned an adhoc task. + $queuedadhoctasks = manager::get_adhoc_tasks(update_object_tags::class); + $this->assertCount(1, $queuedadhoctasks); + + // Ensure the adhoc task spawned has an iteration of 1. + $adhoctask = current($queuedadhoctasks); + $this->assertNotEmpty($adhoctask->get_custom_data()); + $this->assertEquals(1, $adhoctask->get_custom_data()->iteration); + } +} diff --git a/tests/task/update_object_tags_test.php b/tests/task/update_object_tags_test.php new file mode 100644 index 00000000..76fbaccc --- /dev/null +++ b/tests/task/update_object_tags_test.php @@ -0,0 +1,204 @@ +. + +namespace tool_objectfs\task; + +use core\task\manager; +use moodle_exception; +use tool_objectfs\local\tag\tag_manager; +use tool_objectfs\tests\testcase; + +/** + * Tests update_object_tags + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @covers \tool_objectfs\task\update_object_tags + */ +class update_object_tags_test extends testcase { + /** + * Creates object with tags needing to be synced + * @param string $contents contents of object to create. + * @return stdClass object record + */ + private function create_object_needing_tag_sync(string $contents) { + global $DB; + $object = $this->create_duplicated_object($contents); + $DB->set_field('tool_objectfs_objects', 'tagsyncstatus', tag_manager::SYNC_STATUS_NEEDS_SYNC, ['id' => $object->id]); + return $object; + } + + /** + * Tests task exits when the tagging feature is disabled. + */ + public function test_not_enabled() { + $this->resetAfterTest(); + + // By default filesystem does not support and tagging not enabled, so should error. + $task = new update_object_tags(); + + $this->expectException(moodle_exception::class); + $this->expectExceptionMessage(get_string('tagging:migration:notsupported', 'tool_objectfs')); + $task->execute(); + } + + /** + * Tests handles an invalid iteration limit + */ + public function test_invalid_iteration_limit() { + $this->resetAfterTest(); + $this->enable_filesystem_and_set_tagging(true); + + // This should be greater than 1, if zero should error. + set_config('maxtaggingiterations', 0, 'tool_objectfs'); + + // Give it a valid iteration number though. + $task = new update_object_tags(); + $task->set_custom_data(['iteration' => 5]); + + $this->expectException(moodle_exception::class); + $this->expectExceptionMessage(get_string('tagging:migration:invaliditerations', 'tool_objectfs')); + $task->execute(); + } + + /** + * Tests handles an invalid number of iterations in custom data + */ + public function test_invalid_iteration_number() { + $this->resetAfterTest(); + $this->enable_filesystem_and_set_tagging(true); + + // Give it a valid max iteration number. + set_config('maxtaggingiterations', 5, 'tool_objectfs'); + + // But don't set the iteration number on the customdata at all. + $this->expectException(moodle_exception::class); + $this->expectExceptionMessage(get_string('tagging:migration:invaliditerations', 'tool_objectfs')); + + $task = new update_object_tags(); + $task->execute(); + } + + /** + * Tests exits when there are no more objects needing to be synced + */ + public function test_no_more_objects_to_sync() { + $this->resetAfterTest(); + $this->enable_filesystem_and_set_tagging(true); + set_config('maxtaggingiterations', 5, 'tool_objectfs'); + $task = new update_object_tags(); + $task->set_custom_data(['iteration' => 1]); + + // This should not error, only output a string since it is successfully completed. + $this->expectOutputString("No more objects found that need tagging, exiting.\n"); + $task->execute(); + } + + /** + * Tests maxtaggingiterations is correctly checked + */ + public function test_max_iterations() { + $this->resetAfterTest(); + $this->enable_filesystem_and_set_tagging(true); + + // Set max 1 iteration. + set_config('maxtaggingiterations', 1, 'tool_objectfs'); + set_config('maxtaggingperrun', 100, 'tool_objectfs'); + + $task = new update_object_tags(); + + // Give it an iteration number higher. + $task->set_custom_data(['iteration' => 5]); + + $this->expectException(moodle_exception::class); + $this->expectExceptionMessage(get_string('tagging:migration:limitreached', 'tool_objectfs', 5)); + $task->execute(); + } + + /** + * Tests a successful tagging run where it needs to requeue for further processing + */ + public function test_tagging_run_with_requeue() { + $this->resetAfterTest(); + $this->enable_filesystem_and_set_tagging(true); + + // Set max 1 object per run. + set_config('maxtaggingperrun', 1, 'tool_objectfs'); + set_config('maxtaggingiterations', 5, 'tool_objectfs'); + + // Create two objects needing sync. + $this->create_object_needing_tag_sync('object 1'); + $this->create_object_needing_tag_sync('object 2'); + $this->assertCount(2, tag_manager::get_objects_needing_sync(100)); + + $task = new update_object_tags(); + $task->set_custom_data(['iteration' => 1]); + + $this->expectOutputString("Requeing self for another iteration.\n"); + $task->execute(); + + // Ensure that 1 object had its sync status updated. + $this->assertCount(1, tag_manager::get_objects_needing_sync(100)); + + // Ensure there is another task that was re-queued with the iteration incremented. + $tasks = manager::get_adhoc_tasks(update_object_tags::class); + $this->assertCount(1, $tasks); + $task = current($tasks); + $this->assertNotEmpty($task->get_custom_data()); + $this->assertEquals(2, $task->get_custom_data()->iteration); + } + + /** + * Tests get_iteration + * @covers \tool_objectfs\task\update_object_tags::get_iteration + */ + public function test_get_iteration() { + $task = new update_object_tags(); + + // No custom data, should return zero. + $this->assertEquals(0, $task->get_iteration()); + + // Set iteration, it should return that. + $task->set_custom_data([ + 'iteration' => 5, + ]); + $this->assertEquals(5, $task->get_iteration()); + } + + /** + * Tests getting status badge + * @covers \tool_objectfs\task\update_object_tags::get_status_badge + */ + public function test_get_status_badge() { + // Spawn three tasks and break each one in a different way. + // Test their badge output. + $task1 = new update_object_tags(); + $this->assertStringContainsString(get_string('status:waiting', 'tool_objectfs'), + $task1->get_status_badge()); + + $task2 = new update_object_tags(); + $task2->set_fail_delay(1000); + $this->assertStringContainsString(get_string('status:failing', 'tool_objectfs', 1000), + $task2->get_status_badge()); + + $task3 = new update_object_tags(); + $task3->set_timestarted(1000); + $this->assertStringContainsString(get_string('status:running', 'tool_objectfs'), + $task3->get_status_badge()); + } +} diff --git a/version.php b/version.php index ade329a6..4c53cce8 100644 --- a/version.php +++ b/version.php @@ -25,8 +25,8 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2024110800; // The current plugin version (Date: YYYYMMDDXX). -$plugin->release = 2024110800; // Same as version. +$plugin->version = 2024110801; // The current plugin version (Date: YYYYMMDDXX). +$plugin->release = 2024110801; // Same as version. $plugin->requires = 2023042400; // Requires 4.2. $plugin->component = "tool_objectfs"; $plugin->maturity = MATURITY_STABLE; From 074f934501943e1acf8586201ef1a1c699f7fc6d Mon Sep 17 00:00:00 2001 From: Sasha Anastasi Date: Thu, 12 Dec 2024 10:57:36 +1300 Subject: [PATCH 27/28] deleter: manipulator_can_execute check for any falsy value of deletelocal --- classes/local/object_manipulator/deleter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/local/object_manipulator/deleter.php b/classes/local/object_manipulator/deleter.php index 283b9d5c..c0f0b247 100644 --- a/classes/local/object_manipulator/deleter.php +++ b/classes/local/object_manipulator/deleter.php @@ -78,7 +78,7 @@ public function manipulate_object(stdClass $objectrecord) { * @return bool */ protected function manipulator_can_execute() { - if ($this->deletelocal == 0) { + if (!$this->deletelocal) { mtrace("Delete local disabled \n"); return false; } From 0c2b62d998c54d60d9df35864c2a9979c103b03b Mon Sep 17 00:00:00 2001 From: david adamson Date: Mon, 13 Jan 2025 16:57:28 +1100 Subject: [PATCH 28/28] Update for #463: change ci badge, add secret to ci.yml, remove moodle-release.yml --- .github/workflows/ci.yml | 2 ++ .github/workflows/moodle-release.yml | 17 ----------------- README.md | 4 +--- 3 files changed, 3 insertions(+), 20 deletions(-) delete mode 100644 .github/workflows/moodle-release.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ecd6546a..b05968ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,3 +6,5 @@ on: [push, pull_request] jobs: ci: uses: catalyst/catalyst-moodle-workflows/.github/workflows/ci.yml@main + secrets: + moodle_org_token: ${{ secrets.MOODLE_ORG_TOKEN }} diff --git a/.github/workflows/moodle-release.yml b/.github/workflows/moodle-release.yml deleted file mode 100644 index 5d251fbc..00000000 --- a/.github/workflows/moodle-release.yml +++ /dev/null @@ -1,17 +0,0 @@ -# .github/workflows/moodle-release.yml -name: Releasing in the Plugins directory - -on: - push: - branches: - - MOODLE_310_STABLE - paths: - - 'version.php' - -jobs: - release: - uses: catalyst/catalyst-moodle-workflows/.github/workflows/group-310-plus-release.yml@main - with: - plugin_name: tool_objectfs - secrets: - moodle_org_token: ${{ secrets.MOODLE_ORG_TOKEN }} diff --git a/README.md b/README.md index 9e1ce76e..c2832fc5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,4 @@ - - - +[![ci](https://github.com/catalyst/moodle-tool_objectfs/actions/workflows/ci.yml/badge.svg?branch=MOODLE_402_STABLE)](https://github.com/catalyst/moodle-tool_objectfs/actions/workflows/ci.yml?branch=MOODLE_402_STABLE) # moodle-tool_objectfs