From f6b12505a3d45552c1b0450af3ce451763d0ae24 Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Thu, 7 Dec 2023 10:12:43 -0100 Subject: [PATCH] lazy AppConfig Signed-off-by: Maxence Lange --- .../lib/Controller/AppConfigController.php | 2 +- core/Command/Config/ListConfigs.php | 16 +- core/Command/Upgrade.php | 13 +- .../Version29000Date20231126110901.php | 62 + core/ajax/update.php | 6 +- core/register_command.php | 4 +- lib/private/AllConfig.php | 11 +- lib/private/AppConfig.php | 1114 ++++++++++++----- lib/private/Server.php | 24 +- lib/private/SystemConfig.php | 6 +- lib/private/Updater.php | 40 +- lib/private/legacy/OC_App.php | 6 +- lib/public/Exceptions/AppConfigException.php | 34 + .../AppConfigUnknownKeyException.php | 32 + lib/public/IAppConfig.php | 390 +++++- lib/public/IConfig.php | 6 + tests/lib/UpdaterTest.php | 5 + 17 files changed, 1401 insertions(+), 370 deletions(-) create mode 100644 core/Migrations/Version29000Date20231126110901.php create mode 100644 lib/public/Exceptions/AppConfigException.php create mode 100644 lib/public/Exceptions/AppConfigUnknownKeyException.php diff --git a/apps/provisioning_api/lib/Controller/AppConfigController.php b/apps/provisioning_api/lib/Controller/AppConfigController.php index 798daedaf6be8..e74d998261706 100644 --- a/apps/provisioning_api/lib/Controller/AppConfigController.php +++ b/apps/provisioning_api/lib/Controller/AppConfigController.php @@ -113,7 +113,7 @@ public function getKeys(string $app): DataResponse { return new DataResponse(['data' => ['message' => $e->getMessage()]], Http::STATUS_FORBIDDEN); } return new DataResponse([ - 'data' => $this->config->getAppKeys($app), + 'data' => $this->appConfig->getKeys($app), ]); } diff --git a/core/Command/Config/ListConfigs.php b/core/Command/Config/ListConfigs.php index 4adb0a9df5b79..f56d15e33883c 100644 --- a/core/Command/Config/ListConfigs.php +++ b/core/Command/Config/ListConfigs.php @@ -79,7 +79,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int break; case 'all': - $apps = $this->appConfig->getApps(); + $apps = $this->appConfig->getApps(true); $configs = [ 'system' => $this->getSystemConfigs($noSensitiveValues), 'apps' => [], @@ -91,9 +91,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int default: $configs = [ - 'apps' => [ - $app => $this->getAppConfigs($app, $noSensitiveValues), - ], + 'apps' => [$app => $this->getAppConfigs($app, $noSensitiveValues)], ]; } @@ -107,7 +105,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int * @param bool $noSensitiveValues * @return array */ - protected function getSystemConfigs($noSensitiveValues) { + protected function getSystemConfigs(bool $noSensitiveValues): array { $keys = $this->systemConfig->getKeys(); $configs = []; @@ -133,12 +131,8 @@ protected function getSystemConfigs($noSensitiveValues) { * @param bool $noSensitiveValues * @return array */ - protected function getAppConfigs($app, $noSensitiveValues) { - if ($noSensitiveValues) { - return $this->appConfig->getFilteredValues($app, false); - } else { - return $this->appConfig->getValues($app, false); - } + protected function getAppConfigs(string $app, bool $noSensitiveValues) { + return $this->appConfig->getAllValues($app, filtered: $noSensitiveValues); } /** diff --git a/core/Command/Upgrade.php b/core/Command/Upgrade.php index 30acd8f7d4d78..c74b8d270493f 100644 --- a/core/Command/Upgrade.php +++ b/core/Command/Upgrade.php @@ -35,7 +35,6 @@ use OC\Console\TimestampFormatter; use OC\DB\MigratorExecuteSqlEvent; -use OC\Installer; use OC\Repair\Events\RepairAdvanceEvent; use OC\Repair\Events\RepairErrorEvent; use OC\Repair\Events\RepairFinishEvent; @@ -48,7 +47,6 @@ use OCP\EventDispatcher\IEventDispatcher; use OCP\IConfig; use OCP\Util; -use Psr\Log\LoggerInterface; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Helper\ProgressBar; use Symfony\Component\Console\Input\InputInterface; @@ -63,9 +61,7 @@ class Upgrade extends Command { public const ERROR_FAILURE = 5; public function __construct( - private IConfig $config, - private LoggerInterface $logger, - private Installer $installer, + private IConfig $config ) { parent::__construct(); } @@ -91,12 +87,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int } $self = $this; - $updater = new Updater( - $this->config, - \OC::$server->getIntegrityCodeChecker(), - $this->logger, - $this->installer - ); + $updater = \OCP\Server::get(Updater::class); /** @var IEventDispatcher $dispatcher */ $dispatcher = \OC::$server->get(IEventDispatcher::class); diff --git a/core/Migrations/Version29000Date20231126110901.php b/core/Migrations/Version29000Date20231126110901.php new file mode 100644 index 0000000000000..05023c1525034 --- /dev/null +++ b/core/Migrations/Version29000Date20231126110901.php @@ -0,0 +1,62 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Core\Migrations; + +use Closure; +use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +// Create new field in appconfig for the new IAppConfig API, including lazy grouping. +class Version29000Date20231126110901 extends SimpleMigrationStep { + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + + if (!$schema->hasTable('appconfig')) { + return null; + } + + $table = $schema->getTable('appconfig'); + if ($table->hasColumn('lazy_group')) { + return null; + } + + $table->addColumn('lazy_group', Types::STRING, ['length' => 32, 'default' => '']); + $table->addColumn('sensitive', Types::SMALLINT, ['length' => 1, 'default' => '0']); + + if ($table->hasIndex('appconfig_config_key_index')) { + $table->dropIndex('appconfig_config_key_index'); + } + + $table->addIndex(['lazy_group'], 'ac_lazy_i'); + $table->addIndex(['appid', 'lazy_group'], 'ac_app_lazy_i'); + $table->addIndex(['appid', 'lazy_group', 'configkey'], 'ac_app_lazy_key_i'); + + return $schema; + } +} diff --git a/core/ajax/update.php b/core/ajax/update.php index a826b6b7e7a19..af4bb83d95bea 100644 --- a/core/ajax/update.php +++ b/core/ajax/update.php @@ -40,11 +40,14 @@ use OC\Repair\Events\RepairWarningEvent; use OCP\EventDispatcher\Event; use OCP\EventDispatcher\IEventDispatcher; +use OCP\IAppConfig; +use OCP\IConfig; use OCP\IEventSource; use OCP\IEventSourceFactory; use OCP\IL10N; use OCP\ILogger; use OCP\L10N\IFactory; +use OCP\Server; if (!str_contains(@ini_get('disable_functions'), 'set_time_limit')) { @set_time_limit(0); @@ -112,9 +115,10 @@ public function handleRepairFeedback(Event $event): void { \OC_User::setIncognitoMode(true); $logger = \OC::$server->get(\Psr\Log\LoggerInterface::class); - $config = \OC::$server->getConfig(); + $config = Server::get(IConfig::class); $updater = new \OC\Updater( $config, + Server::get(IAppConfig::class), \OC::$server->getIntegrityCodeChecker(), $logger, \OC::$server->query(\OC\Installer::class) diff --git a/core/register_command.php b/core/register_command.php index 27863cfdc8d78..85caba8197155 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -99,7 +99,7 @@ $application->add(new OC\Core\Command\Config\App\GetConfig(\OC::$server->getConfig())); $application->add(new OC\Core\Command\Config\App\SetConfig(\OC::$server->getConfig())); $application->add(new OC\Core\Command\Config\Import(\OC::$server->getConfig())); - $application->add(new OC\Core\Command\Config\ListConfigs(\OC::$server->getSystemConfig(), \OC::$server->getAppConfig())); + $application->add(\OCP\Server::get(\OC\Core\Command\Config\ListConfigs::class)); $application->add(new OC\Core\Command\Config\System\DeleteConfig(\OC::$server->getSystemConfig())); $application->add(new OC\Core\Command\Config\System\GetConfig(\OC::$server->getSystemConfig())); $application->add(new OC\Core\Command\Config\System\SetConfig(\OC::$server->getSystemConfig())); @@ -170,7 +170,7 @@ $application->add(new OC\Core\Command\Maintenance\UpdateHtaccess()); $application->add(new OC\Core\Command\Maintenance\UpdateTheme(\OC::$server->getMimeTypeDetector(), \OC::$server->getMemCacheFactory())); - $application->add(new OC\Core\Command\Upgrade(\OC::$server->getConfig(), \OC::$server->get(LoggerInterface::class), \OC::$server->query(\OC\Installer::class))); + $application->add(new OC\Core\Command\Upgrade(\OC::$server->getConfig())); $application->add(new OC\Core\Command\Maintenance\Repair( new \OC\Repair([], \OC::$server->get(\OCP\EventDispatcher\IEventDispatcher::class), \OC::$server->get(LoggerInterface::class)), \OC::$server->getConfig(), diff --git a/lib/private/AllConfig.php b/lib/private/AllConfig.php index 92178d646352b..529ac0318fdd2 100644 --- a/lib/private/AllConfig.php +++ b/lib/private/AllConfig.php @@ -43,7 +43,6 @@ * Class to combine all the configuration options ownCloud offers */ class AllConfig implements IConfig { - private SystemConfig $systemConfig; private ?IDBConnection $connection = null; /** @@ -68,9 +67,10 @@ class AllConfig implements IConfig { */ private CappedMemoryCache $userCache; - public function __construct(SystemConfig $systemConfig) { + public function __construct( + private SystemConfig $systemConfig + ) { $this->userCache = new CappedMemoryCache(); - $this->systemConfig = $systemConfig; } /** @@ -190,6 +190,7 @@ public function deleteSystemValue($key) { * * @param string $appName the appName that we stored the value under * @return string[] the keys stored for the app + * @deprecated - load IAppConfig directly */ public function getAppKeys($appName) { return \OC::$server->get(AppConfig::class)->getKeys($appName); @@ -201,6 +202,7 @@ public function getAppKeys($appName) { * @param string $appName the appName that we want to store the value under * @param string $key the key of the value, under which will be saved * @param string|float|int $value the value that should be stored + * @deprecated - load IAppConfig directly */ public function setAppValue($appName, $key, $value) { \OC::$server->get(AppConfig::class)->setValue($appName, $key, $value); @@ -213,6 +215,7 @@ public function setAppValue($appName, $key, $value) { * @param string $key the key of the value, under which it was saved * @param string $default the default value to be returned if the value isn't set * @return string the saved value + * @deprecated - load IAppConfig directly */ public function getAppValue($appName, $key, $default = '') { return \OC::$server->get(AppConfig::class)->getValue($appName, $key, $default); @@ -223,6 +226,7 @@ public function getAppValue($appName, $key, $default = '') { * * @param string $appName the appName that we stored the value under * @param string $key the key of the value, under which it was saved + * @deprecated - load IAppConfig directly */ public function deleteAppValue($appName, $key) { \OC::$server->get(AppConfig::class)->deleteKey($appName, $key); @@ -232,6 +236,7 @@ public function deleteAppValue($appName, $key) { * Removes all keys in appconfig belonging to the app * * @param string $appName the appName the configs are stored under + * @deprecated - load IAppConfig directly */ public function deleteAppValues($appName) { \OC::$server->get(AppConfig::class)->deleteApp($appName); diff --git a/lib/private/AppConfig.php b/lib/private/AppConfig.php index 79c650705b23d..8988e0af18267 100644 --- a/lib/private/AppConfig.php +++ b/lib/private/AppConfig.php @@ -1,4 +1,6 @@ * @copyright Copyright (c) 2016, ownCloud, Inc. @@ -9,6 +11,7 @@ * @author Jakob Sack * @author Joas Schilling * @author Jörn Friedrich Dreyer + * @author Maxence Lange * @author michaelletzgus * @author Morris Jobke * @author Robin Appelman @@ -30,354 +33,827 @@ * along with this program. If not, see * */ + namespace OC; -use OC\DB\Connection; -use OC\DB\OracleConnection; +use InvalidArgumentException; +use JsonException; +use OCP\DB\Exception as DBException; use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Exceptions\AppConfigUnknownKeyException; use OCP\IAppConfig; +use OCP\ICache; +use OCP\ICacheFactory; use OCP\IConfig; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; /** * This class provides an easy way for apps to store config values in the * database. + * + * **Note:** since 29.0.0, it supports **lazy grouping** + * + * ### What is lazy grouping ? + * In order to avoid loading useless config values in memory for each request on + * the cloud, it has been made possible to group your config keys. + * Each group, called _lazy group_, is only loaded in memory when one its config + * keys is retrieved. + * + * It is advised to only use the default lazy group, named '' (empty string), for + * config keys used in the registered part of your code that is called even when + * your app is not boot (as in event listeners, ...) + * + * **Warning:** some methods from this class are marked with a warning about ignoring + * lazy grouping, use them wisely and only on part of code called during + * specific request/action + * + * @since 7.0.0 */ class AppConfig implements IAppConfig { - /** @var array[] */ - protected $sensitiveValues = [ - 'circles' => [ - '/^key_pairs$/', - '/^local_gskey$/', - ], - 'external' => [ - '/^sites$/', - ], - 'integration_discourse' => [ - '/^private_key$/', - '/^public_key$/', - ], - 'integration_dropbox' => [ - '/^client_id$/', - '/^client_secret$/', - ], - 'integration_github' => [ - '/^client_id$/', - '/^client_secret$/', - ], - 'integration_gitlab' => [ - '/^client_id$/', - '/^client_secret$/', - '/^oauth_instance_url$/', - ], - 'integration_google' => [ - '/^client_id$/', - '/^client_secret$/', - ], - 'integration_jira' => [ - '/^client_id$/', - '/^client_secret$/', - '/^forced_instance_url$/', - ], - 'integration_onedrive' => [ - '/^client_id$/', - '/^client_secret$/', - ], - 'integration_openproject' => [ - '/^client_id$/', - '/^client_secret$/', - '/^oauth_instance_url$/', - ], - 'integration_reddit' => [ - '/^client_id$/', - '/^client_secret$/', - ], - 'integration_suitecrm' => [ - '/^client_id$/', - '/^client_secret$/', - '/^oauth_instance_url$/', - ], - 'integration_twitter' => [ - '/^consumer_key$/', - '/^consumer_secret$/', - '/^followed_user$/', - ], - 'integration_zammad' => [ - '/^client_id$/', - '/^client_secret$/', - '/^oauth_instance_url$/', - ], - 'notify_push' => [ - '/^cookie$/', - ], - 'spreed' => [ - '/^bridge_bot_password$/', - '/^hosted-signaling-server-(.*)$/', - '/^recording_servers$/', - '/^signaling_servers$/', - '/^signaling_ticket_secret$/', - '/^signaling_token_privkey_(.*)$/', - '/^signaling_token_pubkey_(.*)$/', - '/^sip_bridge_dialin_info$/', - '/^sip_bridge_shared_secret$/', - '/^stun_servers$/', - '/^turn_servers$/', - '/^turn_server_secret$/', - ], - 'support' => [ - '/^last_response$/', - '/^potential_subscription_key$/', - '/^subscription_key$/', - ], - 'theming' => [ - '/^imprintUrl$/', - '/^privacyUrl$/', - '/^slogan$/', - '/^url$/', - ], - 'user_ldap' => [ - '/^(s..)?ldap_agent_password$/', - ], - 'user_saml' => [ - '/^idp-x509cert$/', - ], - ]; - - /** @var Connection */ - protected $conn; - - /** @var array[] */ - private $cache = []; - - /** @var bool */ - private $configLoaded = false; - - /** - * @param Connection $conn - */ - public function __construct(Connection $conn) { - $this->conn = $conn; + private const CACHE_PREFIX = 'core/AppConfig/'; + private const CACHE_TTL = 3600; + private const ALL_APPS_CONFIG = '__ALL__'; + private const APP_MAX_LENGTH = 32; + private const KEY_MAX_LENGTH = 64; + private const LAZY_MAX_LENGTH = 32; + + private array $fastCache = []; // cache for fast config keys + private array $lazyCache = []; // cache for lazy config keys + private array $loaded = []; // loaded lazy group + private ?ICache $distributedCache = null; + + /** @deprecated */ + private bool $configLoaded = false; + /** @deprecated */ + private bool $migrationCompleted = true; + + public function __construct( + ICacheFactory $cacheFactory, + protected IDBConnection $connection, + private LoggerInterface $logger, + ) { + if ($cacheFactory->isAvailable()) { + $this->distributedCache = $cacheFactory->createDistributed(self::CACHE_PREFIX); + } } /** - * @param string $app - * @return array + * @inheritDoc + * @param bool $preloadValues since 29.0.0 preload all values + * + * @return string[] list of app ids + * @since 7.0.0 */ - private function getAppValues($app) { - $this->loadConfigValues(); + public function getApps(bool $preloadValues = false): array { + if ($preloadValues) { + $this->loadConfigAll(); + $keys = array_keys(array_merge($this->fastCache, $this->lazyCache)); + sort($keys); + return $keys; + } + + $qb = $this->connection->getQueryBuilder(); + $qb->selectDistinct('appid') + ->from('appconfig') + ->orderBy('appid', 'asc') + ->groupBy('appid'); + $result = $qb->executeQuery(); - if (isset($this->cache[$app])) { - return $this->cache[$app]; + $rows = $result->fetchAll(); + $apps = []; + foreach ($rows as $row) { + $apps[] = $row['appid']; } - return []; + return $apps; } /** - * Get all apps using the config + * @inheritDoc + * @param string $app id of the app * - * @return string[] an array of app ids + * @return string[] list of stored config keys + * @since 29.0.0 + */ + public function getKeys(string $app): array { + $this->assertParams($app); + $this->loadConfig($app, ignoreLazyGroup: true); + $keys = array_keys($this->cache[$app] ?? []); + sort($keys); + return $keys; + } + + /** + * @inheritDoc + * @param string $app id of the app + * @param string $key config key + * @param string $lazyGroup search key within a lazy group (since 29.0.0) * - * This function returns a list of all apps that have at least one - * entry in the appconfig table. + * @return bool TRUE if key exists + * @since 7.0.0 */ - public function getApps() { - $this->loadConfigValues(); + public function hasKey(string $app, string $key, string $lazyGroup = ''): bool { + $this->assertParams($app, $key, $lazyGroup); + $this->loadConfig($lazyGroup); + ($lazyGroup === '') ? $cache = &$this->fastCache : $cache = &$this->lazyCache; + /** @psalm-suppress UndefinedVariable */ + return isset($cache[$app][$key]); + } - return $this->getSortedKeys($this->cache); + /** + * @param string $app id of the app + * @param string $key config key + * @param string $lazyGroup lazy group + * + * @throws AppConfigUnknownKeyException if config key is not known + * @return bool + * @since 29.0.0 + */ + public function isSensitiveKey(string $app, string $key, string $lazyGroup = ''): bool { + $this->assertParams($app, $key, $lazyGroup); + $this->loadConfig($lazyGroup); + ($lazyGroup === '') ? $cache = &$this->fastCache : $cache = &$this->lazyCache; + /** @psalm-suppress UndefinedVariable */ + return $cache[$app][$key]['sensitive'] ?? throw new AppConfigUnknownKeyException(); } /** - * Get the available keys for an app + * @inheritDoc + * @param string $app if of the app + * @param string $key config key * - * @param string $app the app we are looking for - * @return array an array of key names + * @return string|null lazy group or NULL if key is not found + */ + public function getLazyGroup(string $app, string $key): ?string { + $this->assertParams($app, $key); + $qb = $this->connection->getQueryBuilder(); + $qb->select('lazy_group') + ->from('appconfig') + ->where($qb->expr()->eq('appid', $qb->createNamedParameter($app))) + ->andWhere($qb->expr()->eq('configkey', $qb->createNamedParameter($key))); + $result = $qb->executeQuery(); + $row = $result->fetch(); + $result->closeCursor(); + + return $row['lazy_group'] ?? null; + } + + + /** + * @inheritDoc + * @param string $app id of the app + * @param string $key config keys prefix to search + * @param bool $filtered filter sensitive config values * - * This function gets all keys of an app. Please note that the values are - * not returned. + * @return array [configKey => configValue] + * @since 29.0.0 */ - public function getKeys($app) { - $this->loadConfigValues(); + public function getAllValues(string $app, string $key = '', bool $filtered = false): array { + $this->assertParams($app, $key); + $this->loadConfig(ignoreLazyGroup: true); - if (isset($this->cache[$app])) { - return $this->getSortedKeys($this->cache[$app]); + $values = array_merge($this->fastCache[$app], $this->lazyCache[$app] ?? []); + + /** + * Using the old (deprecated) list of sensitive values. + */ + foreach ($this->getSensitiveKeys($app) as $sensitiveKeyExp) { + $sensitiveKeys = preg_grep($sensitiveKeyExp, array_keys($values)); + foreach ($sensitiveKeys as $sensitiveKey) { + $values[$sensitiveKey]['sensitive'] = true; + } } - return []; + return array_map(function (array $entry) use ($app): mixed { + return ($entry['sensitive']) ? IConfig::SENSITIVE_VALUE : $entry['value']; + }, $values); } - public function getSortedKeys($data) { - $keys = array_keys($data); - sort($keys); - return $keys; + /** + * @inheritDoc + * @param string $key config key + * @param string $lazyGroup lazy group + * + * @return array [appId => configValue] + * @since 29.0.0 + */ + public function searchValues(string $key, string $lazyGroup = ''): array { + $this->assertParams('', $key, $lazyGroup); + $this->loadConfig($lazyGroup); + $values = []; + /** @var array> $cache */ + ($lazyGroup === '') ? $cache = &$this->fastCache : $cache = &$this->lazyCache; + foreach (array_keys($cache) as $app) { + if (isset($cache[$app][$key])) { + $values[$app] = $cache[$app][$key]['value']; + } + } + + return $values; } + /** - * Gets the config value + * @inheritDoc + * @param string $app id of the app + * @param string $key config key + * @param string $default default value (optional) + * @param string $lazyGroup name of the lazy group (optional) * - * @param string $app app - * @param string $key key - * @param string $default = null, default value if the key does not exist - * @return string the value or $default + * @return string stored config value or $default if not set in database + * @throws InvalidArgumentException + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::getValueInt() + * @see self::getValueFloat() + * @see self::getValueBool() + * @see self::getValueArray() + */ + public function getValueString(string $app, string $key, string $default = '', string $lazyGroup = ''): string { + $this->assertParams($app, $key, $lazyGroup); + $this->loadConfig($lazyGroup); + ($lazyGroup === '') ? $cache = &$this->fastCache : $cache = &$this->lazyCache; + /** @psalm-suppress UndefinedVariable */ + return $cache[$app][$key]['value'] ?? $default; + } + + /** + * @inheritDoc + * @param string $app id of the app + * @param string $key config key + * @param int $default default value + * @param string $lazyGroup name of the lazy group * - * This function gets a value from the appconfig table. If the key does - * not exist the default value will be returned + * @return int stored config value or $default if not set in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::getValueString() + * @see self::getValueFloat() + * @see self::getValueBool() + * @see self::getValueArray() */ - public function getValue($app, $key, $default = null) { - $this->loadConfigValues(); + public function getValueInt(string $app, string $key, int $default = 0, string $lazyGroup = ''): int { + return (int) $this->getValueString($app, $key, (string)$default); + } + + /** + * @inheritDoc + * @param string $app id of the app + * @param string $key config key + * @param float $default default value (optional) + * @param string $lazyGroup name of the lazy group (optional) + * + * @return float stored config value or $default if not set in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::getValueString() + * @see self::getValueInt() + * @see self::getValueBool() + * @see self::getValueArray() + */ + public function getValueFloat(string $app, string $key, float $default = 0, string $lazyGroup = ''): float { + return (float) $this->getValueString($app, $key, (string)$default); + } + /** + * @inheritDoc + * @param string $app id of the app + * @param string $key config key + * @param bool $default default value (optional) + * @param string $lazyGroup name of the lazy group (optional) + * + * @return bool stored config value or $default if not set in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::getValueString() + * @see self::getValueInt() + * @see self::getValueFloat() + * @see self::getValueArray() + */ + public function getValueBool(string $app, string $key, bool $default = false, string $lazyGroup = ''): bool { + return in_array($this->getValueString($app, $key, $default ? 'true' : 'false'), ['1', 'true', 'yes', 'on']); + } + + /** + * @inheritDoc + * @param string $app id of the app + * @param string $key config key + * @param array $default default value (optional) + * @param string $lazyGroup name of the lazy group (optional) + * + * @return array stored config value or $default if not set in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::getValueString() + * @see self::getValueInt() + * @see self::getValueFloat() + * @see self::getValueBool() + */ + public function getValueArray(string $app, string $key, array $default = [], string $lazyGroup = ''): array { + try { + $defaultJson = json_encode($default, JSON_THROW_ON_ERROR); + $value = json_decode($this->getValueString($app, $key, $defaultJson), true, JSON_THROW_ON_ERROR); + return (is_array($value)) ? $value : [$value]; + } catch (JsonException) { + return []; + } + } + + /** + * @inheritDoc + * @param string $app id of the app + * @param string $key config key + * @param string $value config value + * @param string $lazyGroup name of the lazy group + * @param bool|null $sensitive value should be hidden when needed. if NULL sensitive flag is not changed in database + * + * @return bool TRUE if value was different, therefor updated in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::setValueInt() + * @see self::setValueFloat() + * @see self::setValueBool() + * @see self::setValueArray() + */ + public function setValueString( + string $app, + string $key, + string $value, + ?bool $sensitive = null, + string $lazyGroup = '' + ): bool { + $this->assertParams($app, $key, $lazyGroup); + $this->loadConfig($lazyGroup); + // store value if not known yet, or value is different, or sensitivity changed + $updated = !$this->hasKey($app, $key, $lazyGroup) + || $value !== $this->getValueString($app, $key, $value, $lazyGroup) + || ($sensitive !== null && $sensitive !== $this->isSensitiveKey($app, $key, $lazyGroup)); + if (!$updated) { + return false; + } + + // update local cache, do not touch sensitive if null or set it to false if new key + ($lazyGroup === '') ? $cache = &$this->fastCache : $cache = &$this->lazyCache; + $cache[$app][$key] = [ + 'value' => $value, + 'sensitive' => $sensitive ?? $cache[$app][$key]['sensitive'] ?? false, + 'lazyGroup' => $lazyGroup + ]; + + $insert = $this->connection->getQueryBuilder(); + $insert->insert('appconfig') + ->setValue('appid', $insert->createNamedParameter($app)) + ->setValue('lazy_group', $insert->createNamedParameter($lazyGroup)) + ->setValue('sensitive', $insert->createNamedParameter(($sensitive ?? false) ? 1 : 0, IQueryBuilder::PARAM_INT)) + ->setValue('configkey', $insert->createNamedParameter($key)) + ->setValue('configvalue', $insert->createNamedParameter($value)); + try { + $insert->executeStatement(); + } catch (DBException $e) { + if ($e->getReason() !== DBException::REASON_UNIQUE_CONSTRAINT_VIOLATION) { + throw $e; // TODO: throw exception or just log and returns false !? + } + + $update = $this->connection->getQueryBuilder(); + $update->update('appconfig') + ->set('configvalue', $update->createNamedParameter($value)) + ->set('lazy_group', $update->createNamedParameter($lazyGroup)) + ->where($update->expr()->eq('appid', $update->createNamedParameter($app))) + ->andWhere($update->expr()->eq('configkey', $update->createNamedParameter($key))); + if ($sensitive !== null) { + $update->set('sensitive', $update->createNamedParameter($sensitive ? 1 : 0, IQueryBuilder::PARAM_INT)); + } + + $update->executeStatement(); + } + + return true; + } + + /** + * @inheritDoc + * @param string $app id of the app + * @param string $key config key + * @param int $value config value + * @param string $lazyGroup name of the lazy group + * @param bool|null $sensitive value should be hidden when needed. if NULL sensitive flag is not changed in database + * + * @return bool TRUE if value was different, therefor updated in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::setValueString() + * @see self::setValueFloat() + * @see self::setValueBool() + * @see self::setValueArray() + */ + public function setValueInt(string $app, string $key, int $value, ?bool $sensitive = null, string $lazyGroup = ''): bool { + return $this->setValueString($app, $key, (string) $value, $sensitive, $lazyGroup); + } + + /** + * @inheritDoc + * @param string $app id of the app + * @param string $key config key + * @param float $value config value + * @param string $lazyGroup name of the lazy group + * @param bool|null $sensitive value should be hidden when needed. if NULL sensitive flag is not changed in database + * + * @return bool TRUE if value was different, therefor updated in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::setValueString() + * @see self::setValueInt() + * @see self::setValueBool() + * @see self::setValueArray() + */ + public function setValueFloat(string $app, string $key, float $value, ?bool $sensitive = null, string $lazyGroup = ''): bool { + return $this->setValueString($app, $key, (string) $value, $sensitive, $lazyGroup); + } + + /** + * @inheritDoc + * @param string $app id of the app + * @param string $key config key + * @param bool $value config value + * @param string $lazyGroup name of the lazy group + * @param bool|null $sensitive value should be hidden when needed. if NULL sensitive flag is not changed in database + * + * @return bool TRUE if value was different, therefor updated in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::setValueString() + * @see self::setValueInt() + * @see self::setValueFloat() + * @see self::setValueArray() + */ + public function setValueBool(string $app, string $key, bool $value, ?bool $sensitive = null, string $lazyGroup = ''): bool { + return $this->setValueString($app, $key, $value ? 'true' : 'false', $sensitive, $lazyGroup); + } + + /** + * @inheritDoc + * @param string $app id of the app + * @param string $key config key + * @param array $value config value + * @param string $lazyGroup name of the lazy group + * @param bool|null $sensitive value should be hidden when needed. if NULL sensitive flag is not changed in database + * + * @return bool TRUE if value was different, therefor updated in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::setValueString() + * @see self::setValueInt() + * @see self::setValueFloat() + * @see self::setValueBool() + */ + public function setValueArray(string $app, string $key, array $value, ?bool $sensitive = null, string $lazyGroup = ''): bool { + try { + return $this->setValueString($app, $key, json_encode($value, JSON_THROW_ON_ERROR), $sensitive, $lazyGroup); + } catch (JsonException $e) { + $this->logger->warning('could not setValueArray', ['app' => $app, 'key' => $key, 'exception' => $e]); + } + + return false; + } + + /** + * @inheritDoc + * @param string $app id of the app + * @param string $key config key + * @param string $lazyGroup name of the lazy group + * + * @since 29.0.0 + */ + public function unsetKey(string $app, string $key): void { + $this->assertParams($app, $key); + $qb = $this->connection->getQueryBuilder(); + $qb->delete('appconfig') + ->where($qb->expr()->eq('appid', $qb->createNamedParameter($app))) + ->andWhere($qb->expr()->eq('configkey', $qb->createNamedParameter($key))); + $qb->executeStatement(); + + // we really want to delete that key + unset($this->lazyCache[$app][$key]); if ($this->hasKey($app, $key)) { - return $this->cache[$app][$key]; + unset($this->fastCache[$app][$key]); + $this->storeDistributedCache(); } + } - return $default; + /** + * @inheritDoc + * @param string $app id of the app + * + * @since 29.0.0 + */ + public function unsetAppKeys(string $app): void { + $this->assertParams($app); + $qb = $this->connection->getQueryBuilder(); + $qb->delete('appconfig') + ->where($qb->expr()->eq('appid', $qb->createNamedParameter($app))); + $qb->executeStatement(); + + $this->clearCache(); } /** - * check if a key is set in the appconfig + * @inheritDoc + * @param string $lazyGroup * - * @param string $app - * @param string $key - * @return bool + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping */ - public function hasKey($app, $key) { - $this->loadConfigValues(); + public function deleteLazyGroup(string $lazyGroup): void { + $this->assertParams(lazyGroup: $lazyGroup); + $qb = $this->connection->getQueryBuilder(); + $qb->delete('appconfig') + ->where($qb->expr()->eq('lazy_group', $qb->createNamedParameter($lazyGroup))); + $qb->executeStatement(); + + $this->clearCache(); + } - return isset($this->cache[$app][$key]); + /** + * @inheritDoc + * @since 29.0.0 + */ + public function clearCache(): void { + $this->lazyCache = $this->fastCache = []; + $this->storeDistributedCache(); } /** - * Sets a value. If the key did not exist before it will be created. + * @return array + * @throws JsonException + */ + public function statusCache(): array { + $distributed = []; + if ($this->distributedCache !== null) { + $distributed['fastCache'] = $this->getDistributedCache('fastCache'); + } + + return [ + 'fastCache' => $this->fastCache, + 'lazyCache' => $this->lazyCache, + 'loaded' => $this->loaded, + 'distributedCache' => $distributed + ]; + } + + /** + * Confirm the string set for app, key and lazyGroup fit the database description * - * @param string $app app - * @param string $key key - * @param string|float|int $value value - * @return bool True if the value was inserted or updated, false if the value was the same + * @param string $app + * @param string $configKey + * @param string $lazyGroup + * @param bool $allowEmptyApp + * @throws InvalidArgumentException */ - public function setValue($app, $key, $value) { - if (!$this->hasKey($app, $key)) { - $inserted = (bool) $this->conn->insertIfNotExist('*PREFIX*appconfig', [ - 'appid' => $app, - 'configkey' => $key, - 'configvalue' => $value, - ], [ - 'appid', - 'configkey', - ]); - - if ($inserted) { - if (!isset($this->cache[$app])) { - $this->cache[$app] = []; - } - - $this->cache[$app][$key] = $value; - return true; - } + private function assertParams(string $app = '', string $configKey = '', string $lazyGroup = '', bool $allowEmptyApp = false): void { + if (!$allowEmptyApp && $app === '') { + throw new InvalidArgumentException('app cannot be an empty string'); + } + if (strlen($app) > self::APP_MAX_LENGTH) { + throw new InvalidArgumentException('Value (' . $app . ') for app is too long (' . self::APP_MAX_LENGTH . ')'); + } + if (strlen($configKey) > self::KEY_MAX_LENGTH) { + throw new InvalidArgumentException('Value (' . $configKey . ') for key is too long (' . self::KEY_MAX_LENGTH . ')'); + } + if (strlen($lazyGroup) > self::LAZY_MAX_LENGTH) { + throw new InvalidArgumentException('Value (' . $lazyGroup . ') for lazyGroup is too long (' . self::LAZY_MAX_LENGTH . ')'); } + } - $sql = $this->conn->getQueryBuilder(); - $sql->update('appconfig') - ->set('configvalue', $sql->createNamedParameter($value)) - ->where($sql->expr()->eq('appid', $sql->createNamedParameter($app))) - ->andWhere($sql->expr()->eq('configkey', $sql->createNamedParameter($key))); - /* - * Only limit to the existing value for non-Oracle DBs: - * http://docs.oracle.com/cd/E11882_01/server.112/e26088/conditions002.htm#i1033286 - * > Large objects (LOBs) are not supported in comparison conditions. + /** + * @param string $item + * + * @return array + * @throws JsonException + */ + private function getDistributedCache(string $item): array { + return json_decode( + $this->distributedCache->get($item), + true, + 32, + JSON_THROW_ON_ERROR + ); + } + + private function loadConfigAll(): void { + // TODO: why use of __ALL__ ? still needed ? + $this->loadConfig(self::ALL_APPS_CONFIG, ignoreLazyGroup: true); + } + + /** + * We store config in multiple internal cache, so we don't load everything + * + * @param string $app + * @param string $lazyGroup + * @param bool $ignoreLazyGroup + */ + private function loadConfig(string $lazyGroup = '', bool $ignoreLazyGroup = false): void { + if ($this->isLoaded($lazyGroup)) { + return; + } + + $qb = $this->connection->getQueryBuilder(); + $qb->from('appconfig'); + /** + * The use of $this->>migrationCompleted is only needed to manage the + * database during the upgrading process to nc29. */ - if (!($this->conn instanceof OracleConnection)) { - /* - * Only update the value when it is not the same - * Note that NULL requires some special handling. Since comparing - * against null can have special results. + if (!$this->migrationCompleted) { + $qb->select('appid', 'configkey', 'configvalue'); + } else { + $qb->select('appid', 'configkey', 'configvalue', 'sensitive', 'lazy_group'); + if (!$ignoreLazyGroup) { + $qb->where($qb->expr()->eq('lazy_group', $qb->createNamedParameter($lazyGroup))); + } + } + + try { + $result = $qb->executeQuery(); + } catch (DBException $e) { + /** + * in case of issue with field name, it means that migration is not completed. + * Falling back to a request without select on lazy_group. + * This whole try/catch and the migrationCompleted variable can be removed in NC30. */ + if ($e->getReason() !== DBException::REASON_INVALID_FIELD_NAME) { + throw $e; + } + + $this->migrationCompleted = false; + $this->loadConfig($lazyGroup, $ignoreLazyGroup); + return; + } - if ($value === null) { - $sql->andWhere( - $sql->expr()->isNotNull('configvalue') - ); - } else { - $sql->andWhere( - $sql->expr()->orX( - $sql->expr()->isNull('configvalue'), - $sql->expr()->neq('configvalue', $sql->createNamedParameter($value), IQueryBuilder::PARAM_STR) - ) - ); + $this->setLoadedStatus($lazyGroup); // in case the group is empty + $rows = $result->fetchAll(); + foreach ($rows as $row) { + if ($ignoreLazyGroup) { + $this->setLoadedStatus($row['lazy_group'] ?? ''); } + // if migration is not completed, 'lazy_group' does not exist in $row + (($row['lazy_group'] ?? '') === '') ? $cache = &$this->fastCache : $cache = &$this->lazyCache; + $cache[$row['appid']][$row['configkey']] = + [ + 'value' => $row['configvalue'], + 'lazyGroup' => $row['lazy_group'], + 'sensitive' => ($row['sensitive'] === 1) + ]; + } + $result->closeCursor(); + if ($lazyGroup === '' && !$ignoreLazyGroup) { + $this->storeDistributedCache(); + } + } + + /** + * @param string $lazyGroup + * + * @return bool + */ + private function isLoaded(string $lazyGroup): bool { + return in_array($lazyGroup, $this->loaded); + } + + private function setLoadedStatus(string $lazyGroup): void { + if ($this->isLoaded($lazyGroup)) { + return; } - $changedRow = (bool) $sql->execute(); + $this->loaded[] = $lazyGroup; + } + + + + private function loadDistributedCache(string $lazyGroup = ''): void { + if ($this->distributedCache === null) { + return; + } - $this->cache[$app][$key] = $value; + if ($lazyGroup !== '') { + return; + } - return $changedRow; + try { + $this->fastCache = $this->getDistributedCache('fastCache'); + $this->setLoadedStatus(''); + } catch (JsonException $e) { + $this->logger->warning('AppConfig distributed cache seems corrupted', ['exception' => $e]); + $this->fastCache = []; + $this->storeDistributedCache(); + } } + /** + * update local cache into distributed system + * + * @param bool $onlyCache + * + * @return void + */ + private function storeDistributedCache(): void { + if ($this->distributedCache === null) { + return; + } + + try { + $fastCache = json_encode($this->fastCache, JSON_THROW_ON_ERROR); + $this->distributedCache->set('fastCache', $fastCache, self::CACHE_TTL); + } catch (JsonException) { + $this->logger->warning('...'); + } + } + + /** + * All methods below this line are set as deprecated. + */ + + /** + * Gets the config value + * + * @param string $app app + * @param string $key key + * @param string $default = null, default value if the key does not exist + * + * @return string the value or $default + * @deprecated - use getValue*() + * + * This function gets a value from the appconfig table. If the key does + * not exist the default value will be returned + */ + public function getValue($app, $key, $default = null) { + $this->loadConfig(); + return $this->fastCache[$app][$key] ?? $default; + } + + /** + * Sets a value. If the key did not exist before it will be created. + * @deprecated + * + * @param string $app app + * @param string $key key + * @param string|float|int $value value + * + * @return bool True if the value was inserted or updated, false if the value was the same + */ + public function setValue($app, $key, $value) { + return $this->setValueString($app, $key, (string)$value); + } + + /** * Deletes a key * * @param string $app app * @param string $key key + * @deprecated use unsetKey() + * @see self::unsetKey() * @return boolean */ public function deleteKey($app, $key) { - $this->loadConfigValues(); - - $sql = $this->conn->getQueryBuilder(); - $sql->delete('appconfig') - ->where($sql->expr()->eq('appid', $sql->createParameter('app'))) - ->andWhere($sql->expr()->eq('configkey', $sql->createParameter('configkey'))) - ->setParameter('app', $app) - ->setParameter('configkey', $key); - $sql->execute(); - - unset($this->cache[$app][$key]); + $this->unsetKey($app, $key); return false; } /** * Remove app from appconfig + * Removes all keys in appconfig belonging to the app. * * @param string $app app - * @return boolean * - * Removes all keys in appconfig belonging to the app. + * @return boolean + * @deprecated use unsetAppKeys() + * @see self::unsetAppKeys() */ public function deleteApp($app) { - $this->loadConfigValues(); - - $sql = $this->conn->getQueryBuilder(); - $sql->delete('appconfig') - ->where($sql->expr()->eq('appid', $sql->createParameter('app'))) - ->setParameter('app', $app); - $sql->execute(); - - unset($this->cache[$app]); + $this->unsetAppKeys($app); return false; } + /** * get multiple values, either the app or key can be used as wildcard by setting it to false * * @param string|false $app * @param string|false $key + * * @return array|false + * @deprecated 29.0.0 use getAllValues() */ public function getValues($app, $key) { if (($app !== false) === ($key !== false)) { return false; } - if ($key === false) { - return $this->getAppValues($app); + if (!$app) { + return $this->searchValues($key); } else { - $appIds = $this->getApps(); - $values = array_map(function ($appId) use ($key) { - return $this->cache[$appId][$key] ?? null; - }, $appIds); - $result = array_combine($appIds, $values); - - return array_filter($result); + return $this->getAllValues($app); } } @@ -385,17 +861,16 @@ public function getValues($app, $key) { * get all values of the app or and filters out sensitive data * * @param string $app + * * @return array + * @deprecated 29.0.0 use getAllFilteredValues() */ public function getFilteredValues($app) { - $values = $this->getValues($app, false); - - if (isset($this->sensitiveValues[$app])) { - foreach ($this->sensitiveValues[$app] as $sensitiveKeyExp) { - $sensitiveKeys = preg_grep($sensitiveKeyExp, array_keys($values)); - foreach ($sensitiveKeys as $sensitiveKey) { - $values[$sensitiveKey] = IConfig::SENSITIVE_VALUE; - } + $values = $this->getAllValues($app, filtered: true); + foreach ($this->getSensitiveKeys($app) as $sensitiveKeyExp) { + $sensitiveKeys = preg_grep($sensitiveKeyExp, array_keys($values)); + foreach ($sensitiveKeys as $sensitiveKey) { + $values[$sensitiveKey] = IConfig::SENSITIVE_VALUE; } } @@ -403,40 +878,121 @@ public function getFilteredValues($app) { } /** - * Load all the app config values + * @param string $app + * + * @return string[] + * @deprecated data sensitivity should be set when calling setValue*() */ - protected function loadConfigValues() { - if ($this->configLoaded) { - return; - } - - $this->cache = []; - - $sql = $this->conn->getQueryBuilder(); - $sql->select('*') - ->from('appconfig'); - $result = $sql->execute(); - - // we are going to store the result in memory anyway - $rows = $result->fetchAll(); - foreach ($rows as $row) { - if (!isset($this->cache[$row['appid']])) { - $this->cache[(string)$row['appid']] = []; - } - - $this->cache[(string)$row['appid']][(string)$row['configkey']] = (string)$row['configvalue']; - } - $result->closeCursor(); - - $this->configLoaded = true; + private function getSensitiveKeys(string $app): array { + $sensitiveValues = [ + 'circles' => [ + '/^key_pairs$/', + '/^local_gskey$/', + ], + 'external' => [ + '/^sites$/', + ], + 'integration_discourse' => [ + '/^private_key$/', + '/^public_key$/', + ], + 'integration_dropbox' => [ + '/^client_id$/', + '/^client_secret$/', + ], + 'integration_github' => [ + '/^client_id$/', + '/^client_secret$/', + ], + 'integration_gitlab' => [ + '/^client_id$/', + '/^client_secret$/', + '/^oauth_instance_url$/', + ], + 'integration_google' => [ + '/^client_id$/', + '/^client_secret$/', + ], + 'integration_jira' => [ + '/^client_id$/', + '/^client_secret$/', + '/^forced_instance_url$/', + ], + 'integration_onedrive' => [ + '/^client_id$/', + '/^client_secret$/', + ], + 'integration_openproject' => [ + '/^client_id$/', + '/^client_secret$/', + '/^oauth_instance_url$/', + ], + 'integration_reddit' => [ + '/^client_id$/', + '/^client_secret$/', + ], + 'integration_suitecrm' => [ + '/^client_id$/', + '/^client_secret$/', + '/^oauth_instance_url$/', + ], + 'integration_twitter' => [ + '/^consumer_key$/', + '/^consumer_secret$/', + '/^followed_user$/', + ], + 'integration_zammad' => [ + '/^client_id$/', + '/^client_secret$/', + '/^oauth_instance_url$/', + ], + 'notify_push' => [ + '/^cookie$/', + ], + 'spreed' => [ + '/^bridge_bot_password$/', + '/^hosted-signaling-server-(.*)$/', + '/^recording_servers$/', + '/^signaling_servers$/', + '/^signaling_ticket_secret$/', + '/^signaling_token_privkey_(.*)$/', + '/^signaling_token_pubkey_(.*)$/', + '/^sip_bridge_dialin_info$/', + '/^sip_bridge_shared_secret$/', + '/^stun_servers$/', + '/^turn_servers$/', + '/^turn_server_secret$/', + ], + 'support' => [ + '/^last_response$/', + '/^potential_subscription_key$/', + '/^subscription_key$/', + ], + 'theming' => [ + '/^imprintUrl$/', + '/^privacyUrl$/', + '/^slogan$/', + '/^url$/', + ], + 'user_ldap' => [ + '/^(s..)?ldap_agent_password$/', + ], + 'user_saml' => [ + '/^idp-x509cert$/', + ], + ]; + + return $sensitiveValues[$app] ?? []; } - /** * Clear all the cached app config values * New cache will be generated next time a config value is retrieved + * + * @deprecated use clearCache(); + * @see self::clearCache() */ public function clearCachedConfig(): void { - $this->configLoaded = false; + $this->clearCache(); } } diff --git a/lib/private/Server.php b/lib/private/Server.php index cf4262e2d5043..c0c0554fe2e88 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -676,27 +676,11 @@ public function __construct($webRoot, \OC\Config $config) { $config = $c->get(\OCP\IConfig::class); if ($config->getSystemValueBool('installed', false) && !(defined('PHPUNIT_RUN') && PHPUNIT_RUN)) { - if (!$config->getSystemValueBool('log_query')) { - try { - $v = \OC_App::getAppVersions(); - } catch (\Doctrine\DBAL\Exception $e) { - // Database service probably unavailable - // Probably related to https://github.com/nextcloud/server/issues/37424 - return $arrayCacheFactory; - } - } else { - // If the log_query is enabled, we can not get the app versions - // as that does a query, which will be logged and the logging - // depends on redis and here we are back again in the same function. - $v = [ - 'log_query' => 'enabled', - ]; + $prefix = $config->getSystemValueString('memcache.prefix', ''); // prefix is updated on update (core or apps) + if ($prefix === '') { + $prefix = md5(\OC_Util::getInstanceId() . '-' . implode(',', \OC_Util::getVersion()) . '-' . \OC::$SERVERROOT); + $config->setSystemValue('memcache.prefix', substr($prefix, 0, -6) . '-' . rand(10000, 99999)); } - $v['core'] = implode(',', \OC_Util::getVersion()); - $version = implode(',', $v); - $instanceId = \OC_Util::getInstanceId(); - $path = \OC::$SERVERROOT; - $prefix = md5($instanceId . '-' . $version . '-' . $path); return new \OC\Memcache\Factory($prefix, $c->get(LoggerInterface::class), $profiler, diff --git a/lib/private/SystemConfig.php b/lib/private/SystemConfig.php index c104f00180916..559847d77785f 100644 --- a/lib/private/SystemConfig.php +++ b/lib/private/SystemConfig.php @@ -123,11 +123,7 @@ class SystemConfig { ], ]; - /** @var Config */ - private $config; - - public function __construct(Config $config) { - $this->config = $config; + public function __construct(private Config $config) { } /** diff --git a/lib/private/Updater.php b/lib/private/Updater.php index 018e4797232ac..036695b6145e8 100644 --- a/lib/private/Updater.php +++ b/lib/private/Updater.php @@ -58,6 +58,7 @@ use OCP\EventDispatcher\Event; use OCP\EventDispatcher\IEventDispatcher; use OCP\HintException; +use OCP\IAppConfig; use OCP\IConfig; use OCP\ILogger; use OCP\Util; @@ -73,18 +74,6 @@ * - failure(string $message) */ class Updater extends BasicEmitter { - /** @var LoggerInterface */ - private $log; - - /** @var IConfig */ - private $config; - - /** @var Checker */ - private $checker; - - /** @var Installer */ - private $installer; - private $logLevelNames = [ 0 => 'Debug', 1 => 'Info', @@ -93,14 +82,12 @@ class Updater extends BasicEmitter { 4 => 'Fatal', ]; - public function __construct(IConfig $config, - Checker $checker, - ?LoggerInterface $log, - Installer $installer) { - $this->log = $log; - $this->config = $config; - $this->checker = $checker; - $this->installer = $installer; + public function __construct( + private IConfig $config, + private IAppConfig $appConfig, + private Checker $checker, + private ?LoggerInterface $log, + private Installer $installer) { } /** @@ -164,6 +151,7 @@ public function upgrade(): bool { $this->emit('\OC\Updater', 'resetLogLevel', [ $logLevel, $this->logLevelNames[$logLevel] ]); $this->config->setSystemValue('loglevel', $logLevel); $this->config->setSystemValue('installed', true); + $this->config->setSystemValue('memcache.prefix', $this->generateCachePrefix()); return $success; } @@ -368,6 +356,10 @@ protected function doAppUpgrade(): void { } } } + + + // setSystemAppValue() ? + } /** @@ -541,4 +533,12 @@ function (MigratorExecuteSqlEvent $event) use ($log): void { $log->info('\OC\Updater::finishedCheckCodeIntegrity: Finished code integrity check', ['app' => 'updater']); }); } + + public function generateCachePrefix(): string { + $this->appConfig->clearCache(); + $v = $this->appConfig->searchValues('installed_version'); + $v['core'] = implode('.', \OC_Util::getVersion()); + $version = implode(':', $v); + return md5(\OC_Util::getInstanceId() . '-' . $version . '-' . \OC::$SERVERROOT); + } } diff --git a/lib/private/legacy/OC_App.php b/lib/private/legacy/OC_App.php index 395c1f44c033e..6e4b40b4165b5 100644 --- a/lib/private/legacy/OC_App.php +++ b/lib/private/legacy/OC_App.php @@ -63,6 +63,7 @@ use OCP\App\ManagerEvent; use OCP\Authentication\IAlternativeLogin; use OCP\EventDispatcher\IEventDispatcher; +use OCP\IAppConfig; use Psr\Container\ContainerExceptionInterface; use Psr\Log\LoggerInterface; @@ -730,8 +731,9 @@ public static function getAppVersions() { static $versions; if (!$versions) { - $appConfig = \OC::$server->getAppConfig(); - $versions = $appConfig->getValues(false, 'installed_version'); + /** @var IAppConfig $appConfig */ + $appConfig = \OCP\Server::get(IAppConfig::class); + $versions = $appConfig->searchValues('installed_version'); } return $versions; } diff --git a/lib/public/Exceptions/AppConfigException.php b/lib/public/Exceptions/AppConfigException.php new file mode 100644 index 0000000000000..73c91d9f018da --- /dev/null +++ b/lib/public/Exceptions/AppConfigException.php @@ -0,0 +1,34 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\Exceptions; + +use Exception; + +/** + * @since 29.0.0 + */ +class AppConfigException extends Exception { +} diff --git a/lib/public/Exceptions/AppConfigUnknownKeyException.php b/lib/public/Exceptions/AppConfigUnknownKeyException.php new file mode 100644 index 0000000000000..e2b9d7fd3dc7a --- /dev/null +++ b/lib/public/Exceptions/AppConfigUnknownKeyException.php @@ -0,0 +1,32 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\Exceptions; + +/** + * @since 29.0.0 + */ +class AppConfigUnknownKeyException extends AppConfigException { +} diff --git a/lib/public/IAppConfig.php b/lib/public/IAppConfig.php index cf387a8a44ccc..8847555500515 100644 --- a/lib/public/IAppConfig.php +++ b/lib/public/IAppConfig.php @@ -1,9 +1,12 @@ * @author Joas Schilling + * @author Maxence Lange * @author Morris Jobke * @author Robin Appelman * @author Robin McCorkell @@ -29,44 +32,401 @@ /** * This class provides an easy way for apps to store config values in the * database. + * + * **Note:** since 29.0.0, it supports **lazy grouping** + * + * ### What is lazy grouping ? + * In order to avoid loading useless config values in memory for each request on + * the cloud, it has been made possible to group your config keys. + * Each group, called _lazy group_, is only loaded in memory when one its config + * keys is retrieved. + * + * It is advised to only use the default lazy group, named '' (empty string), for + * config keys used in the registered part of your code that is called even when + * your app is not boot (as in event listeners, ...) + * + * **Note:** Lazy group are not linked to app ids. Multiple app can share the same + * lazy group and config keys from those apps will be loaded in memory when one value + * from th lazy group is retrieved. + * + * **Warning:** some methods from this class are marked with a warning about ignoring + * lazy grouping, use them wisely and only on part of code called during + * specific request/action + * * @since 7.0.0 */ interface IAppConfig { + /** - * check if a key is set in the appconfig - * @param string $app - * @param string $key - * @return bool + * Get list of all apps that have at least one config value stored in database + * + * **WARNING:** bypass cache and request database each time + * + * @param bool $preloadValues preload all values (since 29.0.0) + * + * @return string[] list of app ids * @since 7.0.0 */ - public function hasKey($app, $key); + public function getApps(bool $preloadValues = false): array; + + /** + * Returns all keys related to an app. + * Please note that the values are not returned. + * + * **Warning:** ignore lazy grouping + * + * @param string $app id of the app + * + * @return string[] list of stored config keys + * @since 29.0.0 + */ + public function getKeys(string $app): array; + + /** + * Check if a key exists in the list of stored config values. + * To search for a key while ignoring lazy grouping, use getLazyGroup() + * + * @param string $app id of the app + * @param string $key config key + * @param string $lazyGroup search key within a lazy group (since 29.0.0) + * + * @return bool TRUE if key exists + * @see self::getLazyGroup() + * @since 7.0.0 + */ + public function hasKey(string $app, string $key, string $lazyGroup = ''): bool; + + /** + * @param string $app id of the app + * @param string $key config key + * @param string $lazyGroup lazy group + * @return bool + * @since 29.0.0 + */ + public function isSensitiveKey(string $app, string $key, string $lazyGroup = ''): bool; + + /** + * Find the lazy group a config key belongs to. + * Returns NULL is key it now known. + * + * **Warning:** bypass cache and request database each time + * + * @param string $app id of the app + * @param string $key config key + * + * @return string|null lazy group or NULL if key is not found + * @since 29.0.0 + */ + public function getLazyGroup(string $app, string $key): ?string; + + /** + * List all config values from an app with config key starting with $key. + * Returns an array with config key as key, stored value as value. + * + * @param string $app id of the app + * @param string $key config keys prefix to search + * @param bool $filtered filter sensitive config values + * + * @return array [configKey => configValue] + * @since 29.0.0 + */ + public function getAllValues(string $app, string $key = '', bool $filtered = false): array; + + /** + * List all apps storing a specific config key and its stored value. + * Returns an array with appId as key, stored value as value. + * + * @param string $key config key + * @param string $lazyGroup lazy group + * + * @return array [appId => configValue] + * @since 29.0.0 + */ + public function searchValues(string $key, string $lazyGroup = ''): array; + + /** + * Get config value assigned to a config key. + * If config key is not found in database, default value is returned. + * If config key belongs to a lazy group, the name of the lazy group needs to be specified. + * + * @param string $app id of the app + * @param string $key config key + * @param string $default default value (optional) + * @param string $lazyGroup name of the lazy group (optional) + * + * @return string stored config value or $default if not set in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::getValueInt() + * @see self::getValueFloat() + * @see self::getValueBool() + * @see self::getValueArray() + */ + public function getValueString(string $app, string $key, string $default = '', string $lazyGroup = ''): string; + + /** + * Get config value assigned to a config key. + * If config key is not found in database, default value is returned. + * If config key belongs to a lazy group, the name of the lazy group needs to be specified. + * + * @param string $app id of the app + * @param string $key config key + * @param int $default default value + * @param string $lazyGroup name of the lazy group + * + * @return int stored config value or $default if not set in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::getValueString() + * @see self::getValueFloat() + * @see self::getValueBool() + * @see self::getValueArray() + */ + public function getValueInt(string $app, string $key, int $default = 0, string $lazyGroup = ''): int; + + /** + * Get config value assigned to a config key. + * If config key is not found in database, default value is returned. + * If config key belongs to a lazy group, the name of the lazy group needs to be specified. + * + * @param string $app id of the app + * @param string $key config key + * @param float $default default value (optional) + * @param string $lazyGroup name of the lazy group (optional) + * + * @return float stored config value or $default if not set in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::getValueString() + * @see self::getValueInt() + * @see self::getValueBool() + * @see self::getValueArray() + */ + public function getValueFloat(string $app, string $key, float $default = 0, string $lazyGroup = ''): float; + + /** + * Get config value assigned to a config key. + * If config key is not found in database, default value is returned. + * If config key belongs to a lazy group, the name of the lazy group needs to be specified. + * + * @param string $app id of the app + * @param string $key config key + * @param bool $default default value (optional) + * @param string $lazyGroup name of the lazy group (optional) + * + * @return bool stored config value or $default if not set in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::getValueString() + * @see self::getValueInt() + * @see self::getValueFloat() + * @see self::getValueArray() + */ + public function getValueBool(string $app, string $key, bool $default = false, string $lazyGroup = ''): bool; + + /** + * Get config value assigned to a config key. + * If config key is not found in database, default value is returned. + * If config key belongs to a lazy group, the name of the lazy group needs to be specified. + * + * @param string $app id of the app + * @param string $key config key + * @param array $default default value (optional) + * @param string $lazyGroup name of the lazy group (optional) + * + * @return array stored config value or $default if not set in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::getValueString() + * @see self::getValueInt() + * @see self::getValueFloat() + * @see self::getValueBool() + */ + public function getValueArray(string $app, string $key, array $default = [], string $lazyGroup = ''): array; + + /** + * Store a config key and its value in database + * If config key is already known with the exact same config value, the database is not updated. + * If config key is not supposed to be read during the boot of the cloud, it is advised to assign it to a lazy group. + * + * @param string $app id of the app + * @param string $key config key + * @param string $value config value + * @param bool|null $sensitive value should be hidden when needed. if NULL sensitive flag is not changed in database + * @param string $lazyGroup name of the lazy group + * + * @return bool TRUE if value was different, therefor updated in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::setValueInt() + * @see self::setValueFloat() + * @see self::setValueBool() + * @see self::setValueArray() + */ + public function setValueString(string $app, string $key, string $value, ?bool $sensitive = null, string $lazyGroup = ''): bool; + + /** + * Store a config key and its value in database + * If config key is already known with the exact same config value, the database is not updated. + * If config key is not supposed to be read during the boot of the cloud, it is advised to assign it to a lazy group. + * + * @param string $app id of the app + * @param string $key config key + * @param int $value config value + * @param bool|null $sensitive value should be hidden when needed. if NULL sensitive flag is not changed in database + * @param string $lazyGroup name of the lazy group + * + * @return bool TRUE if value was different, therefor updated in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::setValueString() + * @see self::setValueFloat() + * @see self::setValueBool() + * @see self::setValueArray() + */ + public function setValueInt(string $app, string $key, int $value, ?bool $sensitive = null, string $lazyGroup = ''): bool; + + /** + * Store a config key and its value in database + * If config key is already known with the exact same config value, the database is not updated. + * If config key is not supposed to be read during the boot of the cloud, it is advised to assign it to a lazy group. + * + * @param string $app id of the app + * @param string $key config key + * @param float $value config value + * @param bool|null $sensitive value should be hidden when needed. if NULL sensitive flag is not changed in database + * @param string $lazyGroup name of the lazy group + * + * @return bool TRUE if value was different, therefor updated in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::setValueString() + * @see self::setValueInt() + * @see self::setValueBool() + * @see self::setValueArray() + */ + public function setValueFloat(string $app, string $key, float $value, ?bool $sensitive = null, string $lazyGroup = ''): bool; + + /** + * Store a config key and its value in database + * If config key is already known with the exact same config value, the database is not updated. + * If config key is not supposed to be read during the boot of the cloud, it is advised to assign it to a lazy group. + * + * @param string $app id of the app + * @param string $key config key + * @param bool $value config value + * @param bool|null $sensitive value should be hidden when needed. if NULL sensitive flag is not changed in database + * @param string $lazyGroup name of the lazy group + * + * @return bool TRUE if value was different, therefor updated in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::setValueString() + * @see self::setValueInt() + * @see self::setValueFloat() + * @see self::setValueArray() + */ + public function setValueBool(string $app, string $key, bool $value, ?bool $sensitive = null, string $lazyGroup = ''): bool; + + /** + * Store a config key and its value in database + * If config key is already known with the exact same config value, the database is not updated. + * If config key is not supposed to be read during the boot of the cloud, it is advised to assign it to a lazy group. + * + * @param string $app id of the app + * @param string $key config key + * @param array $value config value + * @param bool|null $sensitive value should be hidden when needed. if NULL sensitive flag is not changed in database + * @param string $lazyGroup name of the lazy group + * + * @return bool TRUE if value was different, therefor updated in database + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + * @see self::setValueString() + * @see self::setValueInt() + * @see self::setValueFloat() + * @see self::setValueBool() + */ + public function setValueArray(string $app, string $key, array $value, ?bool $sensitive = null, string $lazyGroup = ''): bool; + + /** + * Delete single config key from database. + * + * @param string $app id of the app + * @param string $key config key + * + * @since 29.0.0 + */ + public function unsetKey(string $app, string $key): void; + + /** + * delete all config keys linked to an app + * + * @param string $app id of the app + * @since 29.0.0 + */ + public function unsetAppKeys(string $app): void; + + /** + * delete all config keys linked to a lazy group + * + * @param string $lazyGroup + * + * @since 29.0.0 + * @see IAppConfig for explanation about lazy grouping + */ + public function deleteLazyGroup(string $lazyGroup): void; + + /** + * Clear the cache. + * + * Clearing cache consist of emptying the internal cache and the distributed cache. + * The cache will be rebuilt only the next time a config value is requested. + * + * @since 29.0.0 + */ + public function clearCache(): void; + + /** + * For debug purpose. + * Returns the cached information. + * + * @return array + * @since 29.0.0 + */ + public function statusCache(): array; + + /* + * + * ####################################################################### + * # Below this mark are the method deprecated and replaced since 29.0.0 # + * ####################################################################### + * + */ /** * get multiply values, either the app or key can be used as wildcard by setting it to false + * @deprecated use getAllValues() * * @param string|false $key * @param string|false $app + * * @return array|false * @since 7.0.0 + * @deprecated use getAllValues() + * @see self::getAllValues() */ public function getValues($app, $key); /** * get all values of the app or and filters out sensitive data + * @deprecated use getAllValues() * * @param string $app + * * @return array * @since 12.0.0 + * @see self::getAllValues() */ public function getFilteredValues($app); - - /** - * Get all apps using the config - * @return string[] an array of app ids - * - * This function returns a list of all apps that have at least one - * entry in the appconfig table. - * @since 7.0.0 - */ - public function getApps(); } diff --git a/lib/public/IConfig.php b/lib/public/IConfig.php index 0e7a752321874..fc078aa291c70 100644 --- a/lib/public/IConfig.php +++ b/lib/public/IConfig.php @@ -126,6 +126,7 @@ public function deleteSystemValue($key); * @param string $appName the appName that we stored the value under * @return string[] the keys stored for the app * @since 8.0.0 + * @deprecated - load IAppConfig directly */ public function getAppKeys($appName); @@ -137,6 +138,7 @@ public function getAppKeys($appName); * @param string $value the value that should be stored * @return void * @since 6.0.0 + * @deprecated - load IAppConfig directly */ public function setAppValue($appName, $key, $value); @@ -146,8 +148,10 @@ public function setAppValue($appName, $key, $value); * @param string $appName the appName that we stored the value under * @param string $key the key of the value, under which it was saved * @param string $default the default value to be returned if the value isn't set + * * @return string the saved value * @since 6.0.0 - parameter $default was added in 7.0.0 + * @deprecated - load IAppConfig directly */ public function getAppValue($appName, $key, $default = ''); @@ -157,6 +161,7 @@ public function getAppValue($appName, $key, $default = ''); * @param string $appName the appName that we stored the value under * @param string $key the key of the value, under which it was saved * @since 8.0.0 + * @deprecated - load IAppConfig directly */ public function deleteAppValue($appName, $key); @@ -165,6 +170,7 @@ public function deleteAppValue($appName, $key); * * @param string $appName the appName the configs are stored under * @since 8.0.0 + * @deprecated - load IAppConfig directly */ public function deleteAppValues($appName); diff --git a/tests/lib/UpdaterTest.php b/tests/lib/UpdaterTest.php index 02b2189cdccb8..6040a62574d49 100644 --- a/tests/lib/UpdaterTest.php +++ b/tests/lib/UpdaterTest.php @@ -25,6 +25,7 @@ use OC\Installer; use OC\IntegrityCheck\Checker; use OC\Updater; +use OCP\IAppConfig; use OCP\IConfig; use PHPUnit\Framework\MockObject\MockObject; use Psr\Log\LoggerInterface; @@ -46,6 +47,9 @@ protected function setUp(): void { $this->config = $this->getMockBuilder(IConfig::class) ->disableOriginalConstructor() ->getMock(); + $this->appConfig = $this->getMockBuilder(IAppConfig::class) + ->disableOriginalConstructor() + ->getMock(); $this->logger = $this->getMockBuilder(LoggerInterface::class) ->disableOriginalConstructor() ->getMock(); @@ -58,6 +62,7 @@ protected function setUp(): void { $this->updater = new Updater( $this->config, + $this->appConfig, $this->checker, $this->logger, $this->installer