Merge pull request #272 from splitio/SDKS-7793-flagsets_in_pluggable_…

…storage

[SDKS-7793] Add flag sets support in pluggable storage

CHANGES.txt

@@ -1,3 +1,6 @@
+1.12.0 (December XX, 2023)
+ - Added support for Flag Sets in "consumer" and "partial consumer" modes for pluggable storage.
+
 1.11.0 (November 3, 2023)
   - Added support for Flag Sets on the SDK, which enables grouping feature flags and interacting with the group rather than individually (more details in our documentation):
     - Added new variations of the get treatment methods to support evaluating flags in given flag set/s.

package.json

@@ -22,7 +22,7 @@
     "build": "npm run build:cjs && npm run build:esm",
     "build:esm": "rimraf esm && tsc -m es2015 --outDir esm -d true --declarationDir types",
     "build:cjs": "rimraf cjs && tsc -m CommonJS --outDir cjs",
-    "test": "jest",
+    "test": "jest --runInBand",
     "test:coverage": "jest --coverage",
     "all": "npm run check && npm run build && npm run test",
     "publish:rc": "npm run check && npm run test && npm run build && npm publish --tag rc",

src/logger/messages/warn.ts

@@ -24,7 +24,7 @@ export const codesWarn: [number, string][] = codesError.concat([
   [c.WARN_NOT_EXISTENT_SPLIT, '%s: feature flag "%s" does not exist in this environment. Please double check what feature flags exist in the Split user interface.'],
   [c.WARN_LOWERCASE_TRAFFIC_TYPE, '%s: traffic_type_name should be all lowercase - converting string to lowercase.'],
   [c.WARN_NOT_EXISTENT_TT, '%s: traffic type "%s" does not have any corresponding feature flag in this environment, make sure you\'re tracking your events to a valid traffic type defined in the Split user interface.'],
-  [c.WARN_FLAGSET_NOT_CONFIGURED, '%s: : you passed %s wich is not part of the configured FlagSetsFilter, ignoring Flag Set.'],
+  [c.WARN_FLAGSET_NOT_CONFIGURED, '%s: you passed %s which is not part of the configured FlagSetsFilter, ignoring Flag Set.'],
   // initialization / settings validation
   [c.WARN_INTEGRATION_INVALID, c.LOG_PREFIX_SETTINGS + ': %s integration item(s) at settings is invalid. %s'],
   [c.WARN_SPLITS_FILTER_IGNORED, c.LOG_PREFIX_SETTINGS + ': feature flag filters are not applicable for Consumer modes where the SDK does not keep rollout data in sync. Filters were discarded'],

src/storages/inMemory/SplitsCacheInMemory.ts

@@ -16,9 +16,9 @@ export class SplitsCacheInMemory extends AbstractSplitsCacheSync {
   private splitsWithSegmentsCount: number = 0;
   private flagSetsCache: Record<string, ISet<string>> = {};
 
-  constructor(splitFiltersValidation: ISplitFiltersValidation = { queryString: null, groupedFilters: { bySet: [], byName: [], byPrefix: [] }, validFilters: [] }) {
+  constructor(splitFiltersValidation?: ISplitFiltersValidation) {
     super();
-    this.flagSetsFilter = splitFiltersValidation.groupedFilters.bySet;
+    this.flagSetsFilter = splitFiltersValidation ? splitFiltersValidation.groupedFilters.bySet : [];
   }
 
   clear() {
@@ -114,7 +114,6 @@ export class SplitsCacheInMemory extends AbstractSplitsCacheSync {
       }
     });
     return toReturn;
-
   }
 
   private addToFlagSets(featureFlag: ISplit) {

src/storages/pluggable/SplitsCachePluggable.ts

@@ -2,10 +2,10 @@ import { isFiniteNumber, isNaNNumber } from '../../utils/lang';
 import { KeyBuilder } from '../KeyBuilder';
 import { IPluggableStorageWrapper } from '../types';
 import { ILogger } from '../../logger/types';
-import { ISplit } from '../../dtos/types';
+import { ISplit, ISplitFiltersValidation } from '../../dtos/types';
 import { LOG_PREFIX } from './constants';
 import { AbstractSplitsCacheAsync } from '../AbstractSplitsCacheAsync';
-import { ISet } from '../../utils/lang/sets';
+import { ISet, _Set, returnListDifference } from '../../utils/lang/sets';
 
 /**
  * ISplitsCacheAsync implementation for pluggable storages.
@@ -15,18 +15,20 @@ export class SplitsCachePluggable extends AbstractSplitsCacheAsync {
   private readonly log: ILogger;
   private readonly keys: KeyBuilder;
   private readonly wrapper: IPluggableStorageWrapper;
+  private readonly flagSetsFilter: string[];
 
   /**
    * Create a SplitsCache that uses a storage wrapper.
    * @param log  Logger instance.
    * @param keys  Key builder.
    * @param wrapper  Adapted wrapper storage.
    */
-  constructor(log: ILogger, keys: KeyBuilder, wrapper: IPluggableStorageWrapper) {
+  constructor(log: ILogger, keys: KeyBuilder, wrapper: IPluggableStorageWrapper, splitFiltersValidation?: ISplitFiltersValidation) {
     super();
     this.log = log;
     this.keys = keys;
     this.wrapper = wrapper;
+    this.flagSetsFilter = splitFiltersValidation ? splitFiltersValidation.groupedFilters.bySet : [];
   }
 
   private _decrementCounts(split: ISplit) {
@@ -41,6 +43,24 @@ export class SplitsCachePluggable extends AbstractSplitsCacheAsync {
     return this.wrapper.incr(ttKey);
   }
 
+  private _updateFlagSets(featureFlagName: string, flagSetsOfRemovedFlag?: string[], flagSetsOfAddedFlag?: string[]) {
+    const removeFromFlagSets = returnListDifference(flagSetsOfRemovedFlag, flagSetsOfAddedFlag);
+
+    let addToFlagSets = returnListDifference(flagSetsOfAddedFlag, flagSetsOfRemovedFlag);
+    if (this.flagSetsFilter.length > 0) {
+      addToFlagSets = addToFlagSets.filter(flagSet => {
+        return this.flagSetsFilter.some(filterFlagSet => filterFlagSet === flagSet);
+      });
+    }
+
+    const items = [featureFlagName];
+
+    return Promise.all([
+      ...removeFromFlagSets.map(flagSetName => this.wrapper.removeItems(this.keys.buildFlagSetKey(flagSetName), items)),
+      ...addToFlagSets.map(flagSetName => this.wrapper.addItems(this.keys.buildFlagSetKey(flagSetName), items))
+    ]);
+  }
+
   /**
    * Add a given split.
    * The returned promise is resolved when the operation success
@@ -67,7 +87,7 @@ export class SplitsCachePluggable extends AbstractSplitsCacheAsync {
         return this._incrementCounts(split).then(() => {
           if (parsedPreviousSplit) return this._decrementCounts(parsedPreviousSplit);
         });
-      });
+      }).then(() => this._updateFlagSets(name, parsedPreviousSplit && parsedPreviousSplit.sets, split.sets));
     }).then(() => true);
   }
 
@@ -88,8 +108,9 @@ export class SplitsCachePluggable extends AbstractSplitsCacheAsync {
   removeSplit(name: string) {
     return this.getSplit(name).then((split) => {
       if (split) {
-        this._decrementCounts(split);
+        return this._decrementCounts(split).then(() => this._updateFlagSets(name, split.sets));
       }
+    }).then(() => {
       return this.wrapper.del(this.keys.buildSplitKey(name));
     });
   }
@@ -158,12 +179,19 @@ export class SplitsCachePluggable extends AbstractSplitsCacheAsync {
   /**
    * Get list of split names related to a given flag set names list.
    * The returned promise is resolved with the list of split names,
-   * or rejected if wrapper operation fails.
-   * @todo this is a no-op method to be implemented
+   * or rejected if any wrapper operation fails.
   */
-  getNamesByFlagSets(): Promise<ISet<string>> {
-    this.log.error(LOG_PREFIX + 'ByFlagSet/s evaluations are not supported with pluggable storage yet.');
-    return Promise.reject();
+  getNamesByFlagSets(flagSets: string[]): Promise<ISet<string>> {
+    return Promise.all(flagSets.map(flagSet => {
+      const flagSetKey = this.keys.buildFlagSetKey(flagSet);
+      return this.wrapper.getItems(flagSetKey);
+    })).then(namesByFlagSets => {
+      const featureFlagNames = new _Set<string>();
+      namesByFlagSets.forEach(names => {
+        names.forEach(name => featureFlagNames.add(name));
+      });
+      return featureFlagNames;
+    });
   }
 
   /**

src/storages/pluggable/__tests__/SplitsCachePluggable.spec.ts

@@ -2,8 +2,9 @@ import { SplitsCachePluggable } from '../SplitsCachePluggable';
 import { KeyBuilder } from '../../KeyBuilder';
 import { loggerMock } from '../../../logger/__tests__/sdkLogger.mock';
 import { wrapperMockFactory } from './wrapper.mock';
-import { splitWithUserTT, splitWithAccountTT } from '../../__tests__/testUtils';
+import { splitWithUserTT, splitWithAccountTT, featureFlagOne, featureFlagThree, featureFlagTwo, featureFlagWithEmptyFS, featureFlagWithoutFS } from '../../__tests__/testUtils';
 import { ISplit } from '../../../dtos/types';
+import { _Set } from '../../../utils/lang/sets';
 
 const keysBuilder = new KeyBuilder();
 
@@ -150,4 +151,63 @@ describe('SPLITS CACHE PLUGGABLE', () => {
     expect(await wrapper.getKeysByPrefix('SPLITIO')).toHaveLength(0);
   });
 
+  test('flag set cache tests', async () => {
+    // @ts-ignore
+    const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMockFactory(), { groupedFilters: { bySet: ['o', 'n', 'e', 'x'] } });
+    const emptySet = new _Set([]);
+
+    await cache.addSplits([
+      [featureFlagOne.name, featureFlagOne],
+      [featureFlagTwo.name, featureFlagTwo],
+      [featureFlagThree.name, featureFlagThree],
+    ]);
+    await cache.addSplit(featureFlagWithEmptyFS.name, featureFlagWithEmptyFS);
+
+    expect(await cache.getNamesByFlagSets(['o'])).toEqual(new _Set(['ff_one', 'ff_two']));
+    expect(await cache.getNamesByFlagSets(['n'])).toEqual(new _Set(['ff_one']));
+    expect(await cache.getNamesByFlagSets(['e'])).toEqual(new _Set(['ff_one', 'ff_three']));
+    expect(await cache.getNamesByFlagSets(['t'])).toEqual(emptySet); // 't' not in filter
+    expect(await cache.getNamesByFlagSets(['o', 'n', 'e'])).toEqual(new _Set(['ff_one', 'ff_two', 'ff_three']));
+
+    await cache.addSplit(featureFlagOne.name, { ...featureFlagOne, sets: ['1'] });
+
+    expect(await cache.getNamesByFlagSets(['1'])).toEqual(emptySet); // '1' not in filter
+    expect(await cache.getNamesByFlagSets(['o'])).toEqual(new _Set(['ff_two']));
+    expect(await cache.getNamesByFlagSets(['n'])).toEqual(emptySet);
+
+    await cache.addSplit(featureFlagOne.name, { ...featureFlagOne, sets: ['x'] });
+    expect(await cache.getNamesByFlagSets(['x'])).toEqual(new _Set(['ff_one']));
+    expect(await cache.getNamesByFlagSets(['o', 'e', 'x'])).toEqual(new _Set(['ff_one', 'ff_two', 'ff_three']));
+
+    await cache.removeSplit(featureFlagOne.name);
+    expect(await cache.getNamesByFlagSets(['x'])).toEqual(emptySet);
+
+    await cache.removeSplit(featureFlagOne.name);
+    expect(await cache.getNamesByFlagSets(['y'])).toEqual(emptySet); // 'y' not in filter
+    expect(await cache.getNamesByFlagSets([])).toEqual(emptySet);
+
+    await cache.addSplit(featureFlagWithEmptyFS.name, featureFlagWithoutFS);
+    expect(await cache.getNamesByFlagSets([])).toEqual(emptySet);
+  });
+
+  // if FlagSets filter is not defined, it should store all FlagSets in memory.
+  test('flag set cache tests without filters', async () => {
+    const cacheWithoutFilters = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMockFactory());
+    const emptySet = new _Set([]);
+
+    await cacheWithoutFilters.addSplits([
+      [featureFlagOne.name, featureFlagOne],
+      [featureFlagTwo.name, featureFlagTwo],
+      [featureFlagThree.name, featureFlagThree],
+    ]);
+    await cacheWithoutFilters.addSplit(featureFlagWithEmptyFS.name, featureFlagWithEmptyFS);
+
+    expect(await cacheWithoutFilters.getNamesByFlagSets(['o'])).toEqual(new _Set(['ff_one', 'ff_two']));
+    expect(await cacheWithoutFilters.getNamesByFlagSets(['n'])).toEqual(new _Set(['ff_one']));
+    expect(await cacheWithoutFilters.getNamesByFlagSets(['e'])).toEqual(new _Set(['ff_one', 'ff_three']));
+    expect(await cacheWithoutFilters.getNamesByFlagSets(['t'])).toEqual(new _Set(['ff_two', 'ff_three']));
+    expect(await cacheWithoutFilters.getNamesByFlagSets(['y'])).toEqual(emptySet);
+    expect(await cacheWithoutFilters.getNamesByFlagSets(['o', 'n', 'e'])).toEqual(new _Set(['ff_one', 'ff_two', 'ff_three']));
+  });
+
 });

src/storages/pluggable/index.ts

@@ -105,7 +105,7 @@ export function PluggableStorage(options: PluggableStorageOptions): IStorageAsyn
     });
 
     return {
-      splits: new SplitsCachePluggable(log, keys, wrapper),
+      splits: new SplitsCachePluggable(log, keys, wrapper, settings.sync.__splitFiltersValidation),
       segments: new SegmentsCachePluggable(log, keys, wrapper),
       impressions: isPartialConsumer ? new ImpressionsCacheInMemory(impressionsQueueSize) : new ImpressionsCachePluggable(log, keys.buildImpressionsKey(), wrapper, metadata),
       impressionCounts: impressionCountsCache,

src/storages/types.ts

@@ -44,10 +44,10 @@ export interface IPluggableStorageWrapper {
    *
    * @function del
    * @param {string} key Item to delete
-   * @returns {Promise<void>} A promise that resolves if the operation success, whether the key existed and was removed or it didn't exist.
+   * @returns {Promise<boolean>} A promise that resolves if the operation success, whether the key existed and was removed (resolves with true) or it didn't exist (resolves with false).
    * The promise rejects if the operation fails, for example, if there is a connection error.
    */
-  del: (key: string) => Promise<boolean | void>
+  del: (key: string) => Promise<boolean>
   /**
    * Returns all keys matching the given prefix.
    *

src/types.ts

@@ -197,8 +197,6 @@ interface ISharedSettings {
      * List of feature flag filters. These filters are used to fetch a subset of the feature flag definitions in your environment, in order to reduce the delay of the SDK to be ready.
      * This configuration is only meaningful when the SDK is working in "standalone" mode.
      *
-     * At the moment, only one type of feature flag filter is supported: by name.
-     *
      * Example:
      *  `splitFilter: [
      *    { type: 'byName', values: ['my_feature_flag_1', 'my_feature_flag_2'] }, // will fetch feature flags named 'my_feature_flag_1' and 'my_feature_flag_2'

src/utils/lang/sets.ts

@@ -114,8 +114,16 @@ export const _Set = __getSetConstructor();
 
 export function returnSetsUnion<T>(set: ISet<T>, set2: ISet<T>): ISet<T> {
   const result = new _Set(setToArray(set));
-  set2.forEach( value => {
+  set2.forEach(value => {
     result.add(value);
   });
   return result;
 }
+
+export function returnListDifference<T>(list: T[] = [], list2: T[] = []): T[] {
+  const result = new _Set(list);
+  list2.forEach(item => {
+    result.delete(item);
+  });
+  return setToArray(result);
+}