From 2d6e83afdaf5f1bed846a411cee05d226edabdc9 Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Mon, 26 Apr 2021 17:41:58 -0300 Subject: [PATCH 01/14] wrapper adapter impl and tests --- .eslintrc | 2 +- src/evaluator/value/sanitize.ts | 4 +- .../pluggable/__tests__/wrapper.mock.ts | 98 +++++++++++++ .../__tests__/wrapperAdapter.spec.ts | 136 ++++++++++++++++++ src/storages/pluggable/wrapperAdapter.ts | 87 +++++++++++ src/storages/types.ts | 60 ++++++++ 6 files changed, 384 insertions(+), 3 deletions(-) create mode 100644 src/storages/pluggable/__tests__/wrapper.mock.ts create mode 100644 src/storages/pluggable/__tests__/wrapperAdapter.spec.ts create mode 100644 src/storages/pluggable/wrapperAdapter.ts diff --git a/.eslintrc b/.eslintrc index a44d175d..9231c4df 100644 --- a/.eslintrc +++ b/.eslintrc @@ -66,7 +66,7 @@ "plugin:compat/recommended" ], "rules": { - "no-restricted-syntax": ["error", "ForOfStatement", "ForInStatement", "ArrayPattern"], + "no-restricted-syntax": ["error", "ForOfStatement", "ForInStatement"], "compat/compat": ["error", "defaults, not ie < 10, not node < 6"], "no-throw-literal": "error" }, diff --git a/src/evaluator/value/sanitize.ts b/src/evaluator/value/sanitize.ts index 46cae859..b42ff769 100644 --- a/src/evaluator/value/sanitize.ts +++ b/src/evaluator/value/sanitize.ts @@ -23,12 +23,12 @@ function sanitizeString(val: any): string | undefined { return str ? str : undefined; } -function sanitizeArray(val: any): number[] | string[] | undefined { +function sanitizeArray(val: any): string[] | undefined { const arr = Array.isArray(val) ? uniq(val.map(e => e + '')) : []; return arr.length ? arr : undefined; } -function sanitizeBoolean(val: any): boolean | undefined { +export function sanitizeBoolean(val: any): boolean | undefined { if (val === true || val === false) return val; if (typeof val === 'string') { diff --git a/src/storages/pluggable/__tests__/wrapper.mock.ts b/src/storages/pluggable/__tests__/wrapper.mock.ts new file mode 100644 index 00000000..c84de383 --- /dev/null +++ b/src/storages/pluggable/__tests__/wrapper.mock.ts @@ -0,0 +1,98 @@ +import { startsWith, toNumber } from '../../../utils/lang'; + +let _cache: Record = {}; + +// An in memory ICustomStorageWrapper implementation with Jest mocks +export const wrapperMock = { + + _cache, + + get: jest.fn((key => { + return Promise.resolve(key in _cache ? _cache[key] : null); + })), + set: jest.fn((key: string, value: string) => { + const result = key in _cache; + _cache[key] = value; + return Promise.resolve(result); + }), + del: jest.fn((key: string) => { + const result = key in _cache; + delete _cache[key]; + return Promise.resolve(result); + }), + getKeysByPrefix: jest.fn((prefix: string) => { + return Promise.resolve(Object.keys(_cache).filter(key => startsWith(key, prefix))); + }), + incr: jest.fn((key: string) => { + if (key in _cache) { + const count = toNumber(_cache[key]) + 1; + if (isNaN(count)) return Promise.resolve(false); + _cache[key] = count + ''; + } else { + _cache[key] = '1'; + } + return Promise.resolve(true); + }), + decr: jest.fn((key: string) => { + if (key in _cache) { + const count = toNumber(_cache[key]) - 1; + if (isNaN(count)) return Promise.resolve(false); + _cache[key] = count + ''; + } else { + _cache[key] = '-1'; + } + return Promise.resolve(true); + }), + getMany: jest.fn((keys: string[]) => { + return Promise.resolve(keys.map(key => _cache[key] ? _cache[key] : null)); + }), + // always connects and close + connect: jest.fn(() => Promise.resolve(true)), + close: jest.fn(() => Promise.resolve()), + + mockClear() { + _cache = {}; + this.get.mockClear(); + this.set.mockClear(); + this.del.mockClear(); + this.getKeysByPrefix.mockClear(); + this.incr.mockClear(); + this.decr.mockClear(); + this.getMany.mockClear(); + this.connect.mockClear(); + this.close.mockClear(); + } +}; + +function throwsException() { + throw new Error('some error'); +} + +function rejectedPromise() { + return Promise.reject('some error'); +} + +// @TODO remove if not used +export const wrapperWithExceptions = { + get: throwsException, + set: throwsException, + delete: throwsException, + getKeysByPrefix: throwsException, + incr: throwsException, + decr: throwsException, + getMany: throwsException, + connect: throwsException, + close: throwsException +}; + +export const wrapperWithRejectedPromiseResults = { + get: rejectedPromise, + set: rejectedPromise, + delete: rejectedPromise, + getKeysByPrefix: rejectedPromise, + incr: rejectedPromise, + decr: rejectedPromise, + getMany: rejectedPromise, + connect: rejectedPromise, + close: rejectedPromise +}; diff --git a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts new file mode 100644 index 00000000..c6fe4bfe --- /dev/null +++ b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts @@ -0,0 +1,136 @@ +import { loggerMock } from '../../../logger/__tests__/sdkLogger.mock'; +import thenable from '../../../utils/promise/thenable'; +import { logPrefix } from '../constants'; +import { SplitError } from '../../../utils/lang/errors'; + +/** Mocks */ + +import { wrapperMock } from './wrapper.mock'; // Well implemented wrapper + +function throwsException() { + throw new Error('some error'); +} + +function rejectedPromise() { + return Promise.reject('some error'); +} + +function invalidThenable() { + return { then() { } }; +} + +export const wrapperWithIssues = { + get: undefined, + set: 'invalid value', + del: throwsException, + getKeysByPrefix: rejectedPromise, + incr: invalidThenable, + decr: 'invalid value', + getMany: throwsException, + connect: rejectedPromise, + close: invalidThenable +}; + +export const wrapperWithValuesToSanitize = { + get: () => Promise.resolve(10), + set: () => Promise.resolve('tRue'), + del: () => Promise.resolve(), // no result + getKeysByPrefix: () => Promise.resolve(['1', null, false, true, '2', null]), + incr: () => Promise.resolve('1'), + decr: () => Promise.resolve('0'), + getMany: () => Promise.resolve(['1', null, false, true, '2', null]), + connect: () => Promise.resolve(1), + close: () => Promise.resolve(0), +}; + +const SANITIZED_RESULTS = { + get: '10', + set: true, + del: false, + getKeysByPrefix: ['1', '2'], + incr: false, + decr: false, + getMany: ['1', null, '2', null], + connect: false, + close: false +}; + +const VALID_METHOD_CALLS = { + 'get': ['some_key'], + 'set': ['some_key', 'some_value'], + 'del': ['some_key'], + 'getKeysByPrefix': ['some_prefix'], + 'incr': ['some_key'], + 'decr': ['some_key'], + 'getMany': [['some_key_1', 'some_key_2']], + 'connect': [], + 'close': [] +}; + +// Test target +import { wrapperAdapter } from '../wrapperAdapter'; + +describe('Wrapper Adapter', () => { + + afterEach(() => { + loggerMock.mockClear(); + wrapperMock.mockClear(); + }); + + test('calls original wrapper methods', async () => { + const instance = wrapperAdapter(loggerMock, wrapperMock); + const methods = Object.keys(VALID_METHOD_CALLS); + + for (let i = 0; i < methods.length; i++) { + const method = methods[i]; + const result = instance[method](...VALID_METHOD_CALLS[method]); + expect(wrapperMock[method]).toBeCalledTimes(1); + expect(wrapperMock[method]).toBeCalledWith(...VALID_METHOD_CALLS[method]); + expect(wrapperMock[method]).toHaveReturnedWith(result); + await result; + } + + // no logs if there isn't issues + expect(loggerMock.error).not.toBeCalled(); + expect(loggerMock.warn).not.toBeCalled(); + }); + + test('handle wrapper call exceptions', async () => { // @ts-ignore + const instance = wrapperAdapter(loggerMock, wrapperWithIssues); + const methods = Object.keys(VALID_METHOD_CALLS); + + for (let i = 0; i < methods.length; i++) { + const method = methods[i]; + const result = instance[method](...VALID_METHOD_CALLS[method]); + expect(thenable(result)).toBe(true); + try { + await result; + expect(true).toBe(false); // promise shouldn't be resolved + } catch (e) { + expect(e).toBeInstanceOf(SplitError); + expect(loggerMock.error).toHaveBeenCalledWith(`${logPrefix} wrapper '${method}' operation threw an error. Message: ${e.message}`); + } + } + + expect(loggerMock.error).toBeCalledTimes(methods.length); + }); + + test('sanitize wrapper call results', async () => { // @ts-ignore + const instance = wrapperAdapter(loggerMock, wrapperWithValuesToSanitize); + const methods = Object.keys(VALID_METHOD_CALLS); + + for (let i = 0; i < methods.length; i++) { + try { + const method = methods[i]; + const result = await instance[method](...VALID_METHOD_CALLS[method]); + + expect(result).toEqual(SANITIZED_RESULTS[method]); // result should be sanitized + } catch (e) { + expect(true).toBe(false); // promise shouldn't be rejected + } + } + + expect(loggerMock.warn).toBeCalledTimes(methods.length); // one warning for each wrapper operation call which result had to be sanitized + }); + +}); diff --git a/src/storages/pluggable/wrapperAdapter.ts b/src/storages/pluggable/wrapperAdapter.ts new file mode 100644 index 00000000..f68cf1e4 --- /dev/null +++ b/src/storages/pluggable/wrapperAdapter.ts @@ -0,0 +1,87 @@ +import { toString, isString } from '../../utils/lang'; +import { sanitizeBoolean as sBoolean } from '../../evaluator/value/sanitize'; +import { ILogger } from '../../logger/types'; +import { SplitError } from '../../utils/lang/errors'; +import { ICustomStorageWrapper } from '../types'; +import { logPrefix } from './constants'; + +// Sanitizers return the given value if it is of the expected type, or a new sanitized one if invalid. + +function sanitizeBoolean(val: any): boolean { + return sBoolean(val) || false; +} +sanitizeBoolean.type = 'boolean'; + +function sanitizeArray(val: any): string[] { + if (!Array.isArray(val)) return []; // if not an array, returns a new empty one + if (val.every(isString)) return val; // if all items are valid, return the given array + return val.filter(isString); // otherwise, return a new array filtering the invalid items +} +sanitizeArray.type = 'Array'; + +function sanitizeNullableString(val: any): string | null { + if (val === null) return val; + return toString(val); +} +sanitizeNullableString.type = 'string | null'; + +function sanitizeArrayOfNullableString(val: any): (string | null)[] { + const isStringOrNull = (v: any) => v === null || isString(v); + if (!Array.isArray(val)) return []; + if (val.every(isStringOrNull)) return val; + return val.filter(isStringOrNull); +} +sanitizeArrayOfNullableString.type = 'Array'; + +const METHODS_TO_PROMISE_WRAP: [string, { (val: any): any, type: string }][] = [ + ['get', sanitizeNullableString], + ['set', sanitizeBoolean], + ['del', sanitizeBoolean], + ['getKeysByPrefix', sanitizeArray], + ['incr', sanitizeBoolean], + ['decr', sanitizeBoolean], + ['getMany', sanitizeArrayOfNullableString], + ['connect', sanitizeBoolean], + ['close', sanitizeBoolean] +]; + +/** + * Adapter of the Custom Storage Wrapper. + * Used to properly handle exception and rejected promise results: logs error and wrap them into SplitErrors. + * + * @param log logger instance + * @param wrapper custom storage wrapper to adapt + * @returns an adapted version of the given storage wrapper + */ +export function wrapperAdapter(log: ILogger, wrapper: ICustomStorageWrapper): ICustomStorageWrapper { + + const wrapperAdapter: Record = {}; + + METHODS_TO_PROMISE_WRAP.forEach(([method, sanitizer]) => { + + // Logs error and wraps it into an SplitError object + function handleError(e: any) { + log.error(`${logPrefix} wrapper '${method}' operation threw an error. Message: ${e}`); + return Promise.reject(new SplitError(e)); + } + + wrapperAdapter[method] = function () { + try { + // @ts-ignore + return wrapper[method].apply(wrapper, arguments).then((value => { + const sanitizedValue = sanitizer(value); + // if value had to be sanitized, log a warning + if (sanitizedValue !== value) log.warn(`${logPrefix} Attempted to sanitize return value [${value}] of wrapper '${method}' operation which should be of type [${sanitizer.type}]. Sanitized and processed value => [${sanitizedValue}]`); + + return sanitizedValue; + })).catch(handleError); + } catch (e) { + return handleError(e); + } + }; + + }); + + // @ts-ignore + return wrapperAdapter; +} diff --git a/src/storages/types.ts b/src/storages/types.ts index c4b94ad2..5fe8c1e1 100644 --- a/src/storages/types.ts +++ b/src/storages/types.ts @@ -3,6 +3,66 @@ import { ILogger } from '../logger/types'; import { IReadinessManager } from '../readiness/types'; import { SplitIO, ImpressionDTO } from '../types'; +/** + * Interface to define a custom wrapper storage. + */ +export interface ICustomStorageWrapper { + /** + * Returns a promise that resolves with the element associated with the specified `key`, or null if the key can't be found in the storage. + */ + get: (key: string) => Promise + /** + * Adds or updates an element with a specified `key` and a `value`. + * Returns a promise that resolves with a boolean value: + * - true if the element existed and was updated, + * - or false if the element didn't exist and was added. + */ + set: (key: string, value: string) => Promise + /** + * Removes the specified element by `key`. + * Return a promise that resolves with a boolean value: + * - true if the element existed and has been removed, + * - or false if the element does not exist. + */ + del: (key: string) => Promise + /** + * Returns a promise that resolves with the list of element keys that match with the given `prefix`. + */ + getKeysByPrefix: (prefix: string) => Promise + /** + * Increment in 1 the given `key` value or set it in 1 if the value doesn't exist. + * Return a resolved promise with a boolean value: + * - true if the value was incremented, + * - or false if the value already existed and couldn't be parsed into a finite number. + * @param key + */ + incr: (key: string) => Promise + /** + * Decrement in 1 the given `key` value or set it in -1 if the value doesn't exist. + * Return a promise that resolves with a boolean value: + * - true if the value was decremented, + * - or false if the value already existed and couldn't be parsed into a finite number. + * @param key + */ + decr: (key: string) => Promise + /** + * Returns a promise that resolves with the list of elements associated with the specified list of `keys`. + */ + getMany: (keys: string[]) => Promise<(string | null)[]> + + /** + * For storages that requires to be connected, like database servers. + * Return a promise that resolves with a boolean value: + * - true if the operation succeeded, + * - or false if the operation failed. + */ + connect: () => Promise + /** + * For storages that requires to be closed, for example, to release resources. + */ + close: () => Promise +} + /** Splits cache */ export interface ISplitsCacheBase { From 7c63777bdb96b2f819aa1a24abbac9d1c46cd4bf Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Mon, 26 Apr 2021 18:04:13 -0300 Subject: [PATCH 02/14] update --- src/storages/pluggable/__tests__/wrapperAdapter.spec.ts | 5 +++-- src/storages/pluggable/constants.ts | 1 + src/storages/pluggable/wrapperAdapter.ts | 8 +++++--- 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 src/storages/pluggable/constants.ts diff --git a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts index c6fe4bfe..26c7c17b 100644 --- a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts +++ b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts @@ -1,3 +1,4 @@ +// @ts-nocheck import { loggerMock } from '../../../logger/__tests__/sdkLogger.mock'; import thenable from '../../../utils/promise/thenable'; import { logPrefix } from '../constants'; @@ -95,7 +96,7 @@ describe('Wrapper Adapter', () => { expect(loggerMock.warn).not.toBeCalled(); }); - test('handle wrapper call exceptions', async () => { // @ts-ignore + test('handle wrapper call exceptions', async () => { const instance = wrapperAdapter(loggerMock, wrapperWithIssues); const methods = Object.keys(VALID_METHOD_CALLS); @@ -115,7 +116,7 @@ describe('Wrapper Adapter', () => { expect(loggerMock.error).toBeCalledTimes(methods.length); }); - test('sanitize wrapper call results', async () => { // @ts-ignore + test('sanitize wrapper call results', async () => { const instance = wrapperAdapter(loggerMock, wrapperWithValuesToSanitize); const methods = Object.keys(VALID_METHOD_CALLS); diff --git a/src/storages/pluggable/constants.ts b/src/storages/pluggable/constants.ts new file mode 100644 index 00000000..dbac131e --- /dev/null +++ b/src/storages/pluggable/constants.ts @@ -0,0 +1 @@ +export const logPrefix = 'storage:pluggable:'; diff --git a/src/storages/pluggable/wrapperAdapter.ts b/src/storages/pluggable/wrapperAdapter.ts index f68cf1e4..a442b254 100644 --- a/src/storages/pluggable/wrapperAdapter.ts +++ b/src/storages/pluggable/wrapperAdapter.ts @@ -33,7 +33,7 @@ function sanitizeArrayOfNullableString(val: any): (string | null)[] { } sanitizeArrayOfNullableString.type = 'Array'; -const METHODS_TO_PROMISE_WRAP: [string, { (val: any): any, type: string }][] = [ +const METHODS_TO_PROMISE_WRAP: [string, undefined | { (val: any): any, type: string }][] = [ ['get', sanitizeNullableString], ['set', sanitizeBoolean], ['del', sanitizeBoolean], @@ -42,7 +42,7 @@ const METHODS_TO_PROMISE_WRAP: [string, { (val: any): any, type: string }][] = [ ['decr', sanitizeBoolean], ['getMany', sanitizeArrayOfNullableString], ['connect', sanitizeBoolean], - ['close', sanitizeBoolean] + ['close', undefined] ]; /** @@ -61,7 +61,7 @@ export function wrapperAdapter(log: ILogger, wrapper: ICustomStorageWrapper): IC // Logs error and wraps it into an SplitError object function handleError(e: any) { - log.error(`${logPrefix} wrapper '${method}' operation threw an error. Message: ${e}`); + log.error(`${logPrefix} Wrapper '${method}' operation threw an error. Message: ${e}`); return Promise.reject(new SplitError(e)); } @@ -69,7 +69,9 @@ export function wrapperAdapter(log: ILogger, wrapper: ICustomStorageWrapper): IC try { // @ts-ignore return wrapper[method].apply(wrapper, arguments).then((value => { + if (!sanitizer) return value; const sanitizedValue = sanitizer(value); + // if value had to be sanitized, log a warning if (sanitizedValue !== value) log.warn(`${logPrefix} Attempted to sanitize return value [${value}] of wrapper '${method}' operation which should be of type [${sanitizer.type}]. Sanitized and processed value => [${sanitizedValue}]`); From 111d53cbb85b8e188a32dd7ac8dad8f78a0b5d87 Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Mon, 26 Apr 2021 18:19:17 -0300 Subject: [PATCH 03/14] fixed test --- .../pluggable/__tests__/wrapperAdapter.spec.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts index 26c7c17b..0764c182 100644 --- a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts +++ b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts @@ -40,8 +40,7 @@ export const wrapperWithValuesToSanitize = { incr: () => Promise.resolve('1'), decr: () => Promise.resolve('0'), getMany: () => Promise.resolve(['1', null, false, true, '2', null]), - connect: () => Promise.resolve(1), - close: () => Promise.resolve(0), + connect: () => Promise.resolve(1) }; const SANITIZED_RESULTS = { @@ -52,8 +51,7 @@ const SANITIZED_RESULTS = { incr: false, decr: false, getMany: ['1', null, '2', null], - connect: false, - close: false + connect: false }; const VALID_METHOD_CALLS = { @@ -109,7 +107,7 @@ describe('Wrapper Adapter', () => { expect(true).toBe(false); // promise shouldn't be resolved } catch (e) { expect(e).toBeInstanceOf(SplitError); - expect(loggerMock.error).toHaveBeenCalledWith(`${logPrefix} wrapper '${method}' operation threw an error. Message: ${e.message}`); + expect(loggerMock.error).toHaveBeenCalledWith(`${logPrefix} Wrapper '${method}' operation threw an error. Message: ${e.message}`); } } @@ -118,7 +116,7 @@ describe('Wrapper Adapter', () => { test('sanitize wrapper call results', async () => { const instance = wrapperAdapter(loggerMock, wrapperWithValuesToSanitize); - const methods = Object.keys(VALID_METHOD_CALLS); + const methods = Object.keys(SANITIZED_RESULTS); for (let i = 0; i < methods.length; i++) { try { From e5cd83799211bddbee04b219970691032d256c1e Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Mon, 26 Apr 2021 19:18:52 -0300 Subject: [PATCH 04/14] splits pluggable storage implementation and tests --- src/evaluator/index.ts | 37 ++- src/storages/AbstractSplitsCacheSync.ts | 13 +- src/storages/KeyBuilder.ts | 8 + src/storages/KeyBuilderCS.ts | 4 - src/storages/KeyBuilderSS.ts | 2 +- src/storages/dataLoader.ts | 4 +- .../inLocalStorage/SplitsCacheInLocal.ts | 6 +- src/storages/inMemory/SplitsCacheInMemory.ts | 6 +- src/storages/inRedis/SegmentsCacheInRedis.ts | 1 + src/storages/inRedis/SplitsCacheInRedis.ts | 55 ++-- .../__tests__/EventsCacheInRedis.spec.ts | 2 +- src/storages/inRedis/__tests__/index.spec.ts | 2 +- src/storages/inRedis/index.ts | 12 +- src/storages/metadataBuilder.ts | 10 + .../pluggable/SplitsCachePluggable.ts | 274 ++++++++++++++++++ .../__tests__/SplitsCachePluggable.spec.ts | 180 ++++++++++++ src/storages/types.ts | 16 +- src/sync/polling/syncTasks/splitsSyncTask.ts | 6 +- 18 files changed, 557 insertions(+), 81 deletions(-) create mode 100644 src/storages/metadataBuilder.ts create mode 100644 src/storages/pluggable/SplitsCachePluggable.ts create mode 100644 src/storages/pluggable/__tests__/SplitsCachePluggable.spec.ts diff --git a/src/evaluator/index.ts b/src/evaluator/index.ts index 8dd5aa14..3c12c9b3 100644 --- a/src/evaluator/index.ts +++ b/src/evaluator/index.ts @@ -15,6 +15,14 @@ const treatmentException = { config: null }; +function treatmentsException(splitNames: string[]) { + const evaluations: Record = {}; + splitNames.forEach(splitName => { + evaluations[splitName] = treatmentException; + }); + return evaluations; +} + export function evaluateFeature( log: ILogger, key: SplitIO.SplitKey, @@ -27,10 +35,8 @@ export function evaluateFeature( try { stringifiedSplit = storage.splits.getSplit(splitName); } catch (e) { - // the only scenario where getSplit can throw an error is when the storage - // is redis and there is a connection issue and we can't retrieve the split - // to be evaluated - return Promise.resolve(treatmentException); + // Exception on sync `getSplit` storage. Not possible ATM with InMemory and InLocal storages. + return treatmentException; } if (thenable(stringifiedSplit)) { @@ -40,7 +46,11 @@ export function evaluateFeature( key, attributes, storage, - )); + )).catch( + // Exception on async `getSplit` storage. For example, when the storage is redis or + // pluggable and there is a connection issue and we can't retrieve the split to be evaluated + () => treatmentException + ); } return getEvaluation( @@ -60,22 +70,21 @@ export function evaluateFeatures( storage: IStorageSync | IStorageAsync, ): MaybeThenable> { let stringifiedSplits; - const evaluations: Record = {}; try { stringifiedSplits = storage.splits.getSplits(splitNames); } catch (e) { - // the only scenario where `getSplits` can throw an error is when the storage - // is redis and there is a connection issue and we can't retrieve the split - // to be evaluated - splitNames.forEach(splitName => { - evaluations[splitName] = treatmentException; - }); - return Promise.resolve(evaluations); + // Exception on sync `getSplits` storage. Not possible ATM with InMemory and InLocal storages. + return treatmentsException(splitNames); } return (thenable(stringifiedSplits)) ? - stringifiedSplits.then(splits => getEvaluations(log, splitNames, splits, key, attributes, storage)) : + stringifiedSplits.then(splits => getEvaluations(log, splitNames, splits, key, attributes, storage)) + .catch(() => { + // Exception on async `getSplits` storage. For example, when the storage is redis or + // pluggable and there is a connection issue and we can't retrieve the split to be evaluated + return treatmentsException(splitNames); + }) : getEvaluations(log, splitNames, stringifiedSplits, key, attributes, storage); } diff --git a/src/storages/AbstractSplitsCacheSync.ts b/src/storages/AbstractSplitsCacheSync.ts index 36eef20d..3494f56d 100644 --- a/src/storages/AbstractSplitsCacheSync.ts +++ b/src/storages/AbstractSplitsCacheSync.ts @@ -19,17 +19,10 @@ export default abstract class AbstractSplitsCacheSync implements ISplitsCacheSyn return results; } - abstract removeSplit(name: string): number + abstract removeSplit(name: string): boolean - removeSplits(names: string[]): number { - let len = names.length; - let counter = 0; - - for (let i = 0; i < len; i++) { - counter += this.removeSplit(names[i]); - } - - return counter; + removeSplits(names: string[]): boolean[] { + return names.map(name => this.removeSplit(name)); } abstract getSplit(name: string): string | null diff --git a/src/storages/KeyBuilder.ts b/src/storages/KeyBuilder.ts index 36e4badc..801bab3b 100644 --- a/src/storages/KeyBuilder.ts +++ b/src/storages/KeyBuilder.ts @@ -33,6 +33,14 @@ export default class KeyBuilder { return startsWith(key, `${this.prefix}.split.`); } + buildSplitKeyPrefix() { + return `${this.prefix}.split.`; + } + + buildSplitsWithSegmentCountKey() { + return `${this.prefix}.splits.usingSegments`; + } + buildSegmentNameKey(segmentName: string) { return `${this.prefix}.segment.${segmentName}`; } diff --git a/src/storages/KeyBuilderCS.ts b/src/storages/KeyBuilderCS.ts index 7d2c1312..6bfe7adf 100644 --- a/src/storages/KeyBuilderCS.ts +++ b/src/storages/KeyBuilderCS.ts @@ -26,10 +26,6 @@ export default class KeyBuilderCS extends KeyBuilder { return builtSegmentKeyName.substr(prefix.length); } - buildSplitsWithSegmentCountKey() { - return `${this.prefix}.splits.usingSegments`; - } - buildLastUpdatedKey() { return `${this.prefix}.splits.lastUpdated`; } diff --git a/src/storages/KeyBuilderSS.ts b/src/storages/KeyBuilderSS.ts index 903a0c73..1270c6f3 100644 --- a/src/storages/KeyBuilderSS.ts +++ b/src/storages/KeyBuilderSS.ts @@ -53,7 +53,7 @@ export default class KeyBuilderSS extends KeyBuilder { // } searchPatternForSplitKeys() { - return `${this.prefix}.split.*`; + return `${this.buildSplitKeyPrefix()}*`; } // NOT USED diff --git a/src/storages/dataLoader.ts b/src/storages/dataLoader.ts index 97ae6b3b..26e0090d 100644 --- a/src/storages/dataLoader.ts +++ b/src/storages/dataLoader.ts @@ -39,9 +39,7 @@ export function dataLoaderFactory(preloadedData: SplitIO.PreloadedData): DataLoa storage.splits.setChangeNumber(since); // splitsData in an object where the property is the split name and the pertaining value is a stringified json of its data - Object.keys(splitsData).forEach(splitName => { - storage.splits.addSplit(splitName, splitsData[splitName]); - }); + storage.splits.addSplits(Object.keys(splitsData).map(splitName => [splitName, splitsData[splitName]])); // add mySegments data let mySegmentsData = preloadedData.mySegmentsData && preloadedData.mySegmentsData[userId]; diff --git a/src/storages/inLocalStorage/SplitsCacheInLocal.ts b/src/storages/inLocalStorage/SplitsCacheInLocal.ts index 58d5ba09..56135e57 100644 --- a/src/storages/inLocalStorage/SplitsCacheInLocal.ts +++ b/src/storages/inLocalStorage/SplitsCacheInLocal.ts @@ -119,7 +119,7 @@ export default class SplitsCacheInLocal extends AbstractSplitsCacheSync { } } - removeSplit(name: string): number { + removeSplit(name: string): boolean { try { const split = this.getSplit(name); localStorage.removeItem(this.keys.buildSplitKey(name)); @@ -127,10 +127,10 @@ export default class SplitsCacheInLocal extends AbstractSplitsCacheSync { const parsedSplit = JSON.parse(split as string); this._decrementCounts(parsedSplit); - return 1; + return true; } catch (e) { this.log.error(logPrefix + e); - return 0; + return false; } } diff --git a/src/storages/inMemory/SplitsCacheInMemory.ts b/src/storages/inMemory/SplitsCacheInMemory.ts index 5388d2a4..39eefdcf 100644 --- a/src/storages/inMemory/SplitsCacheInMemory.ts +++ b/src/storages/inMemory/SplitsCacheInMemory.ts @@ -57,7 +57,7 @@ export default class SplitsCacheInMemory extends AbstractSplitsCacheSync { } } - removeSplit(name: string): number { + removeSplit(name: string): boolean { const split = this.getSplit(name); if (split) { // Delete the Split @@ -74,9 +74,9 @@ export default class SplitsCacheInMemory extends AbstractSplitsCacheSync { // Update the segments count. if (usesSegments(parsedSplit)) this.splitsWithSegmentsCount--; - return 1; + return true; } else { - return 0; + return false; } } diff --git a/src/storages/inRedis/SegmentsCacheInRedis.ts b/src/storages/inRedis/SegmentsCacheInRedis.ts index e4b23ccd..73955761 100644 --- a/src/storages/inRedis/SegmentsCacheInRedis.ts +++ b/src/storages/inRedis/SegmentsCacheInRedis.ts @@ -53,6 +53,7 @@ export default class SegmentsCacheInRedis implements ISegmentsCacheAsync { }); } + // @TODO remove: not used and not part of interface registerSegment(segment: string) { return this.registerSegments([segment]); } diff --git a/src/storages/inRedis/SplitsCacheInRedis.ts b/src/storages/inRedis/SplitsCacheInRedis.ts index c601b549..a5ec21c7 100644 --- a/src/storages/inRedis/SplitsCacheInRedis.ts +++ b/src/storages/inRedis/SplitsCacheInRedis.ts @@ -3,6 +3,7 @@ import KeyBuilderSS from '../KeyBuilderSS'; import { ISplitsCacheAsync } from '../types'; import { Redis } from 'ioredis'; import { ILogger } from '../../logger/types'; +import { SplitError } from '../../utils/lang/errors'; const logPrefix = 'storage:redis: '; @@ -17,28 +18,29 @@ function processPipelineAnswer(results: Array<[Error | null, string]>): string[] } /** - * Default ISplitsCacheSync implementation that stores split definitions in memory. - * Supported by all JS runtimes. + * ISplitsCacheAsync implementation that stores split definitions in Redis. + * Supported by Node. */ export default class SplitsCacheInRedis implements ISplitsCacheAsync { private readonly redis: Redis; private readonly keys: KeyBuilderSS; - private redisError?: string; + // private redisError?: string; constructor(private readonly log: ILogger, keys: KeyBuilderSS, redis: Redis) { this.redis = redis; this.keys = keys; - this.redis.on('error', (e) => { - this.redisError = e; - }); + // this.redis.on('error', (e) => { + // this.redisError = e; + // }); - this.redis.on('connect', () => { - this.redisError = undefined; - }); + // this.redis.on('connect', () => { + // this.redisError = undefined; + // }); } + // @TODO fix: incr/decr TT and segments for producer mode. Follow pluggable storage signature addSplit(name: string, split: string): Promise { return this.redis.set( this.keys.buildSplitKey(name), split @@ -47,6 +49,7 @@ export default class SplitsCacheInRedis implements ISplitsCacheAsync { ); } + // @TODO fix: incr/decr TT and segments for producer mode. Follow pluggable storage signature addSplits(entries: [string, string][]): Promise { if (entries.length) { const cmds = entries.map(keyValuePair => ['set', this.keys.buildSplitKey(keyValuePair[0]), keyValuePair[1]]); @@ -60,17 +63,22 @@ export default class SplitsCacheInRedis implements ISplitsCacheAsync { } } + // @TODO implement for producer mode. Follow pluggable storage signature + killLocally(): Promise { + throw new Error('Method not implemented.'); + } + /** * Remove a given split from Redis. Returns the number of deleted keys. */ - removeSplit(name: string): Promise { + removeSplit(name: string): Promise { return this.redis.del(this.keys.buildSplitKey(name)); } /** * Bulk delete of splits from Redis. Returns the number of deleted keys. */ - removeSplits(names: string[]): Promise { + removeSplits(names: string[]): Promise { if (names.length) { return this.redis.del(names.map(n => this.keys.buildSplitKey(n))); } else { @@ -80,13 +88,15 @@ export default class SplitsCacheInRedis implements ISplitsCacheAsync { /** * Get split definition or null if it's not defined. + * Returned promise is Rejected with an SplitError if redis operation fails. */ getSplit(name: string): Promise { - if (this.redisError) { - this.log.error(logPrefix + this.redisError); + // @TODO remove next block if not required + // if (this.redisError) { + // this.log.error(logPrefix + this.redisError); - throw this.redisError; - } + // throw this.redisError; + // } return this.redis.get(this.keys.buildSplitKey(name)); } @@ -135,6 +145,8 @@ export default class SplitsCacheInRedis implements ISplitsCacheAsync { // If there is a number there should be > 0, otherwise the TT is considered as not existent. return this.redis.get(this.keys.buildTrafficTypeKey(trafficType)) .then((ttCount: string | null | number) => { + if (ttCount === null) return false; // if entry doesn't exist, means that TT doesn't exist + ttCount = parseInt(ttCount as string, 10); if (!isFiniteNumber(ttCount) || ttCount < 0) { this.log.info(logPrefix + `Could not validate traffic type existance of ${trafficType} due to data corruption of some sorts.`); @@ -166,13 +178,16 @@ export default class SplitsCacheInRedis implements ISplitsCacheAsync { /** * Fetches multiple splits definitions. + * Returned promise is Rejected with an SplitError if redis operation fails. */ getSplits(names: string[]): Promise> { - if (this.redisError) { - this.log.error(logPrefix + this.redisError); + // @TODO remove next block if not required + // if (this.redisError) { + // this.log.error(logPrefix + this.redisError); + + // throw this.redisError; + // } - throw this.redisError; - } const splits: Record = {}; const keys = names.map(name => this.keys.buildSplitKey(name)); return this.redis.mget(...keys) @@ -184,7 +199,7 @@ export default class SplitsCacheInRedis implements ISplitsCacheAsync { }) .catch(e => { this.log.error(logPrefix + `Could not grab splits due to an error: ${e}.`); - return Promise.reject(e); + return Promise.reject(new SplitError(e)); }); } diff --git a/src/storages/inRedis/__tests__/EventsCacheInRedis.spec.ts b/src/storages/inRedis/__tests__/EventsCacheInRedis.spec.ts index c4384726..f5831481 100644 --- a/src/storages/inRedis/__tests__/EventsCacheInRedis.spec.ts +++ b/src/storages/inRedis/__tests__/EventsCacheInRedis.spec.ts @@ -4,7 +4,7 @@ import isEqual from 'lodash/isEqual'; import { loggerMock } from '../../../logger/__tests__/sdkLogger.mock'; import KeyBuilderSS from '../../KeyBuilderSS'; import EventsCacheInRedis from '../EventsCacheInRedis'; -import { metadataBuilder } from '../index'; +import { metadataBuilder } from '../../metadataBuilder'; const prefix = 'events_cache_ut'; const metadata = { version: 'js_someversion', ip: 'some_ip', hostname: 'some_hostname' }; diff --git a/src/storages/inRedis/__tests__/index.spec.ts b/src/storages/inRedis/__tests__/index.spec.ts index fa62acf3..2ac012f1 100644 --- a/src/storages/inRedis/__tests__/index.spec.ts +++ b/src/storages/inRedis/__tests__/index.spec.ts @@ -1,4 +1,4 @@ -import { metadataBuilder } from '../index'; +import { metadataBuilder } from '../../metadataBuilder'; import { UNKNOWN } from '../../../utils/constants'; import { IMetadata } from '../../../dtos/types'; diff --git a/src/storages/inRedis/index.ts b/src/storages/inRedis/index.ts index 52093236..935cdf67 100644 --- a/src/storages/inRedis/index.ts +++ b/src/storages/inRedis/index.ts @@ -1,6 +1,5 @@ import RedisAdapter from './RedisAdapter'; import { IStorageAsync, IStorageFactoryParams } from '../types'; -import { IMetadata, IRedisMetadata } from '../../dtos/types'; import KeyBuilderSS from '../KeyBuilderSS'; import SplitsCacheInRedis from './SplitsCacheInRedis'; import SegmentsCacheInRedis from './SegmentsCacheInRedis'; @@ -8,23 +7,14 @@ import ImpressionsCacheInRedis from './ImpressionsCacheInRedis'; import EventsCacheInRedis from './EventsCacheInRedis'; import LatenciesCacheInRedis from './LatenciesCacheInRedis'; import CountsCacheInRedis from './CountsCacheInRedis'; -import { UNKNOWN } from '../../utils/constants'; import { SDK_SPLITS_ARRIVED, SDK_SEGMENTS_ARRIVED } from '../../readiness/constants'; +import { metadataBuilder } from '../metadataBuilder'; export interface InRedisStorageOptions { prefix?: string options?: Record } -// exported for testing purposes -export function metadataBuilder(metadata: IMetadata): IRedisMetadata { - return { - s: metadata.version, - i: metadata.ip || UNKNOWN, - n: metadata.hostname || UNKNOWN, - }; -} - /** * InRedis storage factory for consumer server-side SplitFactory, that uses `Ioredis` Redis client for Node. * @see {@link https://www.npmjs.com/package/ioredis} diff --git a/src/storages/metadataBuilder.ts b/src/storages/metadataBuilder.ts new file mode 100644 index 00000000..ca75572d --- /dev/null +++ b/src/storages/metadataBuilder.ts @@ -0,0 +1,10 @@ +import { IMetadata, IRedisMetadata } from '../dtos/types'; +import { UNKNOWN } from '../utils/constants'; + +export function metadataBuilder(metadata: IMetadata): IRedisMetadata { + return { + s: metadata.version, + i: metadata.ip || UNKNOWN, + n: metadata.hostname || UNKNOWN, + }; +} diff --git a/src/storages/pluggable/SplitsCachePluggable.ts b/src/storages/pluggable/SplitsCachePluggable.ts new file mode 100644 index 00000000..a1866150 --- /dev/null +++ b/src/storages/pluggable/SplitsCachePluggable.ts @@ -0,0 +1,274 @@ +import { isFiniteNumber, toNumber, isNaNNumber } from '../../utils/lang'; +import KeyBuilder from '../KeyBuilder'; +import { ICustomStorageWrapper, ISplitsCacheAsync } from '../types'; +import { ILogger } from '../../logger/types'; +import { usesSegments } from '../AbstractSplitsCacheSync'; +import { ISplit } from '../../dtos/types'; +import { logPrefix } from './constants'; +import { SplitError } from '../../utils/lang/errors'; + +/** + * ISplitsCacheAsync implementation for pluggable storages. + */ +export class SplitsCachePluggable implements ISplitsCacheAsync { + + private readonly log: ILogger; + private readonly keys: KeyBuilder; + private readonly wrapper: ICustomStorageWrapper; + + /** + * + * @param log logger instance + * @param keys key builder + * @param wrapper adapted wrapper storage + */ + constructor(log: ILogger, keys: KeyBuilder, wrapper: ICustomStorageWrapper) { + this.log = log; + this.keys = keys; + this.wrapper = wrapper; + } + + private _decrementCounts(split: ISplit) { + const promises = []; + if (split.trafficTypeName) { + const ttKey = this.keys.buildTrafficTypeKey(split.trafficTypeName); + promises.push(this.wrapper.decr(ttKey)); + } + + if (usesSegments(split)) { + const segmentsCountKey = this.keys.buildSplitsWithSegmentCountKey(); + promises.push(this.wrapper.decr(segmentsCountKey)); + } + + return Promise.all(promises); + } + + private _incrementCounts(split: ISplit) { + const promises = []; + if (split.trafficTypeName) { + const ttKey = this.keys.buildTrafficTypeKey(split.trafficTypeName); + promises.push(this.wrapper.incr(ttKey)); + } + + if (usesSegments(split)) { + const segmentsCountKey = this.keys.buildSplitsWithSegmentCountKey(); + promises.push(this.wrapper.incr(segmentsCountKey)); + } + + return Promise.all(promises); + } + + /** + * Add a given split. + * The returned promise is resolved when the operation success + * or rejected with an SplitError if it fails (e.g., wrapper operation fails) + */ + addSplit(name: string, split: string): Promise { + const splitKey = this.keys.buildSplitKey(name); + return this.wrapper.get(splitKey).then(splitFromStorage => { + + // handling parsing error as SplitErrors + let parsedPreviousSplit, parsedSplit; + try { + parsedPreviousSplit = splitFromStorage ? JSON.parse(splitFromStorage) : undefined; + parsedSplit = JSON.parse(split); + } catch (e) { + throw new SplitError('Error parsing split definition: ' + e); + } + + return Promise.all([ + // If it's an update, we decrement the traffic type and segment count of the existing split, + parsedPreviousSplit && this._decrementCounts(parsedPreviousSplit), + this.wrapper.set(splitKey, split), + this._incrementCounts(parsedSplit) + ]); + }).then(() => true); + } + + /** + * Add a list of splits. + * The returned promise is resolved when the operation success + * or rejected with an SplitError if it fails (e.g., wrapper operation fails) + */ + addSplits(entries: [string, string][]): Promise { + return Promise.all(entries.map(keyValuePair => this.addSplit(keyValuePair[0], keyValuePair[1]))); + } + + /** + * Remove a given split. + * The returned promise is resolved when the operation success, with a boolean indicating if the split existed or not. + * or rejected with an SplitError if it fails (e.g., wrapper operation fails). + */ + removeSplit(name: string): Promise { + return this.getSplit(name).then((split) => { + if (split) { + const parsedSplit = JSON.parse(split); + this._decrementCounts(parsedSplit); + } + return this.wrapper.del(this.keys.buildSplitKey(name)); + }); + } + + /** + * Remove a list of splits. + * The returned promise is resolved when the operation success, with a boolean array indicating if the splits existed or not. + * or rejected with an SplitError if it fails (e.g., wrapper operation fails). + */ + removeSplits(names: string[]): Promise { + return Promise.all(names.map(name => this.removeSplit(name))); + } + + /** + * Get split. + * The returned promise is resolved with the split definition or null if it's not defined, + * or rejected with an SplitError if wrapper operation fails. + */ + getSplit(name: string): Promise { + return this.wrapper.get(this.keys.buildSplitKey(name)); + } + + /** + * Get list of splits. + * The returned promise is resolved with a map of split names to their split definition or null if it's not defined, + * or rejected with an SplitError if wrapper operation fails. + */ + getSplits(names: string[]): Promise> { + const keys = names.map(name => this.keys.buildSplitKey(name)); + + return this.wrapper.getMany(keys).then(splitDefinitions => { + const splits: Record = {}; + names.forEach((name, idx) => { + splits[name] = splitDefinitions[idx]; + }); + return Promise.resolve(splits); + }); + } + + /** + * Get list of all split definitions. + * The returned promise is resolved with the list of split definitions, + * or rejected with an SplitError if wrapper operation fails. + */ + getAll(): Promise { + return this.wrapper.getKeysByPrefix(this.keys.buildSplitKeyPrefix()).then( + (listOfKeys) => Promise.all(listOfKeys.map(this.wrapper.get) as Promise[]) + ); + } + + /** + * Get list of split names. + * The returned promise is resolved with the list of split names, + * or rejected with an SplitError if wrapper operation fails. + */ + getSplitNames(): Promise { + return this.wrapper.getKeysByPrefix(this.keys.buildSplitKeyPrefix()).then( + (listOfKeys) => listOfKeys.map(this.keys.extractKey) + ); + } + + /** + * Check traffic type existance. + * The returned promise is resolved with a boolean indicating whether the TT exist or not. + * In case of wrapper operation failures, the promise resolves with a true value, assuming that the TT might exist. + * It will never be rejected. + */ + trafficTypeExists(trafficType: string): Promise { + // If there is a number there should be > 0, otherwise the TT is considered as not existent. + return this.wrapper.get(this.keys.buildTrafficTypeKey(trafficType)) + .then((ttCount: string | null | number) => { + if (ttCount === null) return false; // if entry doesn't exist, means that TT doesn't exist + + ttCount = parseInt(ttCount as string, 10); + if (!isFiniteNumber(ttCount) || ttCount < 0) { + this.log.info(logPrefix + `Could not validate traffic type existance of ${trafficType} due to data corruption of some sorts.`); + return false; + } + + return ttCount > 0; + }).catch(e => { + this.log.error(logPrefix + `Could not validate traffic type existance of ${trafficType} due to an error: ${e}.`); + // If there is an error, bypass the validation so the event can get tracked. + return true; + }); + } + + /** + * Set till number. + * The returned promise is resolved when the operation success, + * or rejected with an SplitError if it fails (e.g., wrapper operation fails). + */ + setChangeNumber(changeNumber: number): Promise { + return this.wrapper.set(this.keys.buildSplitsTillKey(), changeNumber + ''); + } + + /** + * Get till number or null if it's not defined. + * The returned promise is resolved with the changeNumber or -1 if it doesn't exist or a wrapper operation fails. + * The promise will never be rejected. + */ + getChangeNumber(): Promise { + return this.wrapper.get(this.keys.buildSplitsTillKey()).then((value) => { + const i = parseInt(value as string, 10); + + return isNaNNumber(i) ? -1 : i; + }).catch((e) => { + this.log.error(logPrefix + 'Could not retrieve changeNumber from storage. Error: ' + e); + return -1; + }); + } + + // @TODO revisit segment-related methods ('usesSegments', 'getRegisteredSegments', 'registerSegments') + usesSegments(): Promise { + return this.wrapper.get(this.keys.buildSplitsWithSegmentCountKey()) + .then(storedCount => { + const splitsWithSegmentsCount = storedCount ? toNumber(storedCount) : 0; + if (isFiniteNumber(splitsWithSegmentsCount)) { + return splitsWithSegmentsCount > 0; + } else { + return true; // If stored valus is invalid, assume we need them. + } + }).catch(() => true); // If wrapper operation fails, assume we need them. + } + + // @TODO remove `clear` from SplitsStorage + clear(): Promise { + return Promise.resolve(true); + } + + /** + * Check if the splits information is already stored in cache. + * Noop, just keeping the interface. This is used by client-side implementations only. + */ + checkCache(): Promise { + return Promise.resolve(true); + } + + /** + * Kill `name` split and set `defaultTreatment` and `changeNumber`. + * Used for SPLIT_KILL push notifications. + * + * @param {string} name + * @param {string} defaultTreatment + * @param {number} changeNumber + * @returns {Promise} a promise that is resolved once the split kill operation is performed. The fulfillment value is a boolean: `true` if the kill success updating the split or `false` if no split is updated, + * for instance, if the `changeNumber` is old, or if the split is not found (e.g., `/splitchanges` hasn't been fetched yet), or if the storage fails to apply the update. + * The promise will never be rejected. + */ + killLocally(name: string, defaultTreatment: string, changeNumber: number): Promise { + return this.getSplit(name).then(split => { + + if (split) { + const parsedSplit: ISplit = JSON.parse(split); + if (!parsedSplit.changeNumber || parsedSplit.changeNumber < changeNumber) { + parsedSplit.killed = true; + parsedSplit.defaultTreatment = defaultTreatment; + parsedSplit.changeNumber = changeNumber; + const newSplit = JSON.stringify(parsedSplit); + + return this.addSplit(name, newSplit); + } + } + return false; + }).catch(() => false); + } +} diff --git a/src/storages/pluggable/__tests__/SplitsCachePluggable.spec.ts b/src/storages/pluggable/__tests__/SplitsCachePluggable.spec.ts new file mode 100644 index 00000000..6e3c1c3e --- /dev/null +++ b/src/storages/pluggable/__tests__/SplitsCachePluggable.spec.ts @@ -0,0 +1,180 @@ +import { SplitsCachePluggable } from '../SplitsCachePluggable'; +import KeyBuilder from '../../KeyBuilder'; +import { loggerMock } from '../../../logger/__tests__/sdkLogger.mock'; +import { wrapperMock } from './wrapper.mock'; + +const splitWithUserTT = '{ "trafficTypeName": "user_tt" }'; +const splitWithAccountTT = '{ "trafficTypeName": "account_tt" }'; +const splitWithAccountTTAndUsesSegments = '{ "trafficTypeName": "account_tt", "conditions": [{ "matcherGroup": { "matchers": [{ "matcherType": "IN_SEGMENT" }]}}] }'; +const keysBuilder = new KeyBuilder(); + +describe('SPLITS CACHE PLUGGABLE', () => { + + afterEach(() => { + loggerMock.mockClear(); + wrapperMock.mockClear(); + }); + + test('add/remove/get splits', async () => { + const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMock); + + // Assert addSplit and addSplits + await cache.addSplits([ + ['lol1', splitWithUserTT], + ['lol2', splitWithAccountTT] + ]); + await cache.addSplit('lol3', splitWithAccountTT); + + // adding malformed or existing splits with the same definitions will not have effects + await expect(cache.addSplits([ + ['lol1', splitWithUserTT], + ['lol2', '{ /'] + ])).rejects.toBeTruthy(); + await expect(cache.addSplit('lol3', '{ /')).rejects.toBeTruthy(); + + // Assert getAll + let values = await cache.getAll(); + + expect(values.length).toBe(3); + expect(values.indexOf(splitWithUserTT) !== -1).toBe(true); + expect(values.indexOf(splitWithAccountTT) !== -1).toBe(true); + + // Assert getSplits + let valuesObj = await cache.getSplits(['lol2', 'lol3']); + + expect(Object.keys(valuesObj).length).toBe(2); + expect(valuesObj.lol2).toBe(splitWithAccountTT); + expect(valuesObj.lol3).toBe(splitWithAccountTT); + + // Assert getSplitNames + let splitNames = await cache.getSplitNames(); + + expect(splitNames.length).toBe(3); + expect(splitNames.indexOf('lol1') !== -1).toBe(true); + expect(splitNames.indexOf('lol2') !== -1).toBe(true); + expect(splitNames.indexOf('lol3') !== -1).toBe(true); + + // Assert removeSplit + await cache.removeSplit('lol1'); + + values = await cache.getAll(); + expect(values.length).toBe(2); + expect(await cache.getSplit('lol1')).toBe(null); + expect(await cache.getSplit('lol2')).toBe(splitWithAccountTT); + + // Assert removeSplits + await cache.addSplit('lol1', splitWithUserTT); + await cache.removeSplits(['lol1', 'lol3']); + + values = await cache.getAll(); + expect(values.length).toBe(1); + splitNames = await cache.getSplitNames(); + expect(splitNames.length).toBe(1); + expect(await cache.getSplit('lol1')).toBe(null); + expect(await cache.getSplit('lol2')).toBe(splitWithAccountTT); + + }); + + test('set/get change number', async () => { + const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMock); + + expect(await cache.getChangeNumber()).toBe(-1); // if not set yet, changeNumber is -1 + await cache.setChangeNumber(123); + expect(await cache.getChangeNumber()).toBe(123); + + }); + + test('trafficTypeExists', async () => { + const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMock); + + await cache.addSplits([ + ['split1', splitWithUserTT], + ['split2', splitWithAccountTT], + ['split3', splitWithUserTT], + ['malformed', '{}'] + ]); + await cache.addSplit('split4', splitWithUserTT); + await cache.addSplit('split4', splitWithUserTT); // trying to add the same definition for an already added split will not have effect + + expect(await cache.trafficTypeExists('user_tt')).toBe(true); + expect(await cache.trafficTypeExists('account_tt')).toBe(true); + expect(await cache.trafficTypeExists('not_existent_tt')).toBe(false); + + await cache.removeSplit('split4'); + + expect(await cache.trafficTypeExists('user_tt')).toBe(true); + expect(await cache.trafficTypeExists('account_tt')).toBe(true); + + await cache.removeSplits(['split3', 'split2']); // it'll invoke a loop of removeSplit + + expect(await cache.trafficTypeExists('user_tt')).toBe(true); + expect(await cache.trafficTypeExists('account_tt')).toBe(false); + + await cache.removeSplit('split1'); + + expect(await cache.trafficTypeExists('user_tt')).toBe(false); + expect(await cache.trafficTypeExists('account_tt')).toBe(false); + + await cache.addSplit('split1', splitWithUserTT); + expect(await cache.trafficTypeExists('user_tt')).toBe(true); + + await cache.addSplit('split1', splitWithAccountTT); + expect(await cache.trafficTypeExists('account_tt')).toBe(true); + expect(await cache.trafficTypeExists('user_tt')).toBe(false); + + }); + + test('usesSegments', async () => { + const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMock); + + await cache.addSplits([['split1', splitWithUserTT], ['split2', splitWithAccountTT],]); + expect(await cache.usesSegments()).toBe(false); // 0 splits using segments + + await cache.addSplit('split3', splitWithAccountTTAndUsesSegments); + expect(await cache.usesSegments()).toBe(true); // 1 split using segments + + await cache.addSplit('split4', splitWithAccountTTAndUsesSegments); + expect(await cache.usesSegments()).toBe(true); // 2 splits using segments + + await cache.removeSplit('split3'); + expect(await cache.usesSegments()).toBe(true); // 1 split using segments + + await cache.removeSplit('split4'); + expect(await cache.usesSegments()).toBe(false); // 0 splits using segments + }); + + test('killLocally', async () => { + const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMock); + + await cache.addSplit('lol1', splitWithUserTT); + await cache.addSplit('lol2', splitWithAccountTT); + const initialChangeNumber = await cache.getChangeNumber(); + + // kill an non-existent split + let updated = await cache.killLocally('nonexistent_split', 'other_treatment', 101); + const nonexistentSplit = await cache.getSplit('nonexistent_split'); + + expect(updated).toBe(false); // killLocally resolves without update if split doesn't exist + expect(nonexistentSplit).toBe(null); // non-existent split keeps being non-existent + + // kill an existent split + updated = await cache.killLocally('lol1', 'some_treatment', 100); + let lol1Split = JSON.parse(await cache.getSplit('lol1') as string); + + expect(updated).toBe(true); // killLocally resolves with update if split is changed + expect(lol1Split.killed).toBe(true); // existing split must be killed + expect(lol1Split.defaultTreatment).toBe('some_treatment'); // existing split must have new default treatment + expect(lol1Split.changeNumber).toBe(100); // existing split must have the given change number + expect(await cache.getChangeNumber()).toBe(initialChangeNumber); // cache changeNumber is not changed + + // not update if changeNumber is old + updated = await cache.killLocally('lol1', 'some_treatment_2', 90); + lol1Split = JSON.parse(await cache.getSplit('lol1') as string); + + expect(updated).toBe(false); // killLocally resolves without update if changeNumber is old + expect(lol1Split.defaultTreatment).not.toBe('some_treatment_2'); // existing split is not updated if given changeNumber is older + + }); + +}); + diff --git a/src/storages/types.ts b/src/storages/types.ts index 5fe8c1e1..ecdf9f11 100644 --- a/src/storages/types.ts +++ b/src/storages/types.ts @@ -66,10 +66,10 @@ export interface ICustomStorageWrapper { /** Splits cache */ export interface ISplitsCacheBase { - addSplit(name: string, split: string): MaybeThenable, + addSplit(name: string, split: string): MaybeThenable, // @TODO remove as in spec addSplits(entries: [string, string][]): MaybeThenable, - removeSplit(name: string): MaybeThenable, - removeSplits(names: string[]): MaybeThenable, + removeSplit(name: string): MaybeThenable, // @TODO remove as in spec + removeSplits(names: string[]): MaybeThenable, getSplit(name: string): MaybeThenable, getSplits(names: string[]): MaybeThenable>, // `fetchMany` in spec setChangeNumber(changeNumber: number): MaybeThenable, @@ -80,13 +80,14 @@ export interface ISplitsCacheBase { usesSegments(): MaybeThenable, clear(): MaybeThenable, checkCache(): MaybeThenable, + killLocally(name: string, defaultTreatment: string, changeNumber: number): MaybeThenable } export interface ISplitsCacheSync extends ISplitsCacheBase { addSplit(name: string, split: string): boolean, addSplits(entries: [string, string][]): boolean[] - removeSplit(name: string): number - removeSplits(names: string[]): number + removeSplit(name: string): boolean + removeSplits(names: string[]): boolean[] getSplit(name: string): string | null getSplits(names: string[]): Record setChangeNumber(changeNumber: number): boolean @@ -103,8 +104,8 @@ export interface ISplitsCacheSync extends ISplitsCacheBase { export interface ISplitsCacheAsync extends ISplitsCacheBase { addSplit(name: string, split: string): Promise, addSplits(entries: [string, string][]): Promise, - removeSplit(name: string): Promise, - removeSplits(names: string[]): Promise, + removeSplit(name: string): Promise, + removeSplits(names: string[]): Promise, getSplit(name: string): Promise, getSplits(names: string[]): Promise>, setChangeNumber(changeNumber: number): Promise, @@ -115,6 +116,7 @@ export interface ISplitsCacheAsync extends ISplitsCacheBase { usesSegments(): Promise, clear(): Promise, checkCache(): Promise, + killLocally(name: string, defaultTreatment: string, changeNumber: number): Promise } /** Segments cache */ diff --git a/src/sync/polling/syncTasks/splitsSyncTask.ts b/src/sync/polling/syncTasks/splitsSyncTask.ts index a69e579a..aec19fe5 100644 --- a/src/sync/polling/syncTasks/splitsSyncTask.ts +++ b/src/sync/polling/syncTasks/splitsSyncTask.ts @@ -1,6 +1,6 @@ import { SplitError } from '../../../utils/lang/errors'; import { _Set, setToArray, ISet } from '../../../utils/lang/sets'; -import { ISegmentsCacheSync, ISplitsCacheSync, IStorageSync } from '../../../storages/types'; +import { ISegmentsCacheSync, ISplitsCacheBase, IStorageSync } from '../../../storages/types'; import { ISplitChangesFetcher } from '../fetchers/types'; import { ISplit, ISplitChangesResponse } from '../../../dtos/types'; import { IReadinessManager, ISplitsEventEmitter } from '../../../readiness/types'; @@ -84,7 +84,7 @@ export function computeSplitsMutation(entries: ISplit[]): ISplitMutations { export function splitChangesUpdaterFactory( log: ILogger, splitChangesFetcher: ISplitChangesFetcher, - splitsCache: ISplitsCacheSync, + splitsCache: ISplitsCacheBase, segmentsCache: ISegmentsCacheSync, splitsEventEmitter: ISplitsEventEmitter, requestTimeoutBeforeReady: number, @@ -129,7 +129,7 @@ export function splitChangesUpdaterFactory( log.debug(SYNC_SPLITS_SEGMENTS, [mutation.segments.length]); // Write into storage - // @TODO if allowing custom storages, wrap errors as SplitErrors to distinguish from user callback errors + // @TODO in producer mode, keep consistency of wrapped storate by calling `setChangeNumber` if the other wrapper operations have succeeded return Promise.all([ // calling first `setChangenumber` method, to perform cache flush if split filter queryString changed splitsCache.setChangeNumber(splitChanges.till), From 2c7f89a26bb571548d33e9cf712a44207a5244a3 Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Tue, 27 Apr 2021 21:21:35 -0300 Subject: [PATCH 05/14] polishing --- src/storages/AbstractSplitsCacheSync.ts | 8 +------- src/storages/inRedis/SplitsCacheInRedis.ts | 2 +- src/storages/pluggable/SplitsCachePluggable.ts | 4 ++-- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/storages/AbstractSplitsCacheSync.ts b/src/storages/AbstractSplitsCacheSync.ts index 3494f56d..3c0c251e 100644 --- a/src/storages/AbstractSplitsCacheSync.ts +++ b/src/storages/AbstractSplitsCacheSync.ts @@ -10,13 +10,7 @@ export default abstract class AbstractSplitsCacheSync implements ISplitsCacheSyn abstract addSplit(name: string, split: string): boolean; addSplits(entries: [string, string][]): boolean[] { - const results: boolean[] = []; - - entries.forEach(keyValuePair => { - results.push(this.addSplit(keyValuePair[0], keyValuePair[1])); - }); - - return results; + return entries.map(keyValuePair => this.addSplit(keyValuePair[0], keyValuePair[1])); } abstract removeSplit(name: string): boolean diff --git a/src/storages/inRedis/SplitsCacheInRedis.ts b/src/storages/inRedis/SplitsCacheInRedis.ts index a5ec21c7..c9694862 100644 --- a/src/storages/inRedis/SplitsCacheInRedis.ts +++ b/src/storages/inRedis/SplitsCacheInRedis.ts @@ -113,7 +113,7 @@ export default class SplitsCacheInRedis implements ISplitsCacheAsync { } /** - * Get till number or null if it's not defined. + * Get till number or -1 if it's not defined. * * @TODO pending error handling */ diff --git a/src/storages/pluggable/SplitsCachePluggable.ts b/src/storages/pluggable/SplitsCachePluggable.ts index a1866150..68e9486b 100644 --- a/src/storages/pluggable/SplitsCachePluggable.ts +++ b/src/storages/pluggable/SplitsCachePluggable.ts @@ -202,7 +202,7 @@ export class SplitsCachePluggable implements ISplitsCacheAsync { } /** - * Get till number or null if it's not defined. + * Get till number or -1 if it's not defined. * The returned promise is resolved with the changeNumber or -1 if it doesn't exist or a wrapper operation fails. * The promise will never be rejected. */ @@ -230,7 +230,7 @@ export class SplitsCachePluggable implements ISplitsCacheAsync { }).catch(() => true); // If wrapper operation fails, assume we need them. } - // @TODO remove `clear` from SplitsStorage + // @TODO implement for DataLoader/Producer mode clear(): Promise { return Promise.resolve(true); } From 7447709045077bc7e944aa16d6f33aca52de3d2e Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Tue, 27 Apr 2021 23:26:36 -0300 Subject: [PATCH 06/14] added remaining wrapper methods --- .../pluggable/__tests__/wrapper.mock.ts | 58 ++++++++----------- .../__tests__/wrapperAdapter.spec.ts | 34 +++++++++-- src/storages/pluggable/wrapperAdapter.ts | 15 ++++- src/storages/types.ts | 38 ++++++++++-- 4 files changed, 100 insertions(+), 45 deletions(-) diff --git a/src/storages/pluggable/__tests__/wrapper.mock.ts b/src/storages/pluggable/__tests__/wrapper.mock.ts index c84de383..1f8af875 100644 --- a/src/storages/pluggable/__tests__/wrapper.mock.ts +++ b/src/storages/pluggable/__tests__/wrapper.mock.ts @@ -1,11 +1,13 @@ import { startsWith, toNumber } from '../../../utils/lang'; let _cache: Record = {}; +let _queues: Record = {}; // An in memory ICustomStorageWrapper implementation with Jest mocks export const wrapperMock = { _cache, + _queues, get: jest.fn((key => { return Promise.resolve(key in _cache ? _cache[key] : null); @@ -15,6 +17,11 @@ export const wrapperMock = { _cache[key] = value; return Promise.resolve(result); }), + getAndSet: jest.fn((key: string, value: string) => { + const result = key in _cache ? _cache[key] : null; + _cache[key] = value; + return Promise.resolve(result); + }), del: jest.fn((key: string) => { const result = key in _cache; delete _cache[key]; @@ -23,6 +30,9 @@ export const wrapperMock = { getKeysByPrefix: jest.fn((prefix: string) => { return Promise.resolve(Object.keys(_cache).filter(key => startsWith(key, prefix))); }), + getByPrefix: jest.fn((prefix: string) => { + return Promise.resolve(Object.keys(_cache).filter(key => startsWith(key, prefix)).map(key => _cache[key])); + }), incr: jest.fn((key: string) => { if (key in _cache) { const count = toNumber(_cache[key]) + 1; @@ -46,6 +56,21 @@ export const wrapperMock = { getMany: jest.fn((keys: string[]) => { return Promise.resolve(keys.map(key => _cache[key] ? _cache[key] : null)); }), + pushItems: jest.fn((key: string, items: string[]) => { + if (!(key in _queues)) _queues[key] = []; + _queues[key].push(...items); + return Promise.resolve(true); + }), + popItems: jest.fn((key: string, count: number) => { + return Promise.resolve(key in _queues ? _queues[key].splice(0, count) : []); + }), + getItemsCount: jest.fn((key: string) => { + return Promise.resolve(key in _queues ? _queues[key].length : 0); + }), + itemContains: jest.fn((key: string, item: string) => { + return Promise.resolve(key in _queues && _queues[key].indexOf(item) > -1 ? true : false); + }), + // always connects and close connect: jest.fn(() => Promise.resolve(true)), close: jest.fn(() => Promise.resolve()), @@ -63,36 +88,3 @@ export const wrapperMock = { this.close.mockClear(); } }; - -function throwsException() { - throw new Error('some error'); -} - -function rejectedPromise() { - return Promise.reject('some error'); -} - -// @TODO remove if not used -export const wrapperWithExceptions = { - get: throwsException, - set: throwsException, - delete: throwsException, - getKeysByPrefix: throwsException, - incr: throwsException, - decr: throwsException, - getMany: throwsException, - connect: throwsException, - close: throwsException -}; - -export const wrapperWithRejectedPromiseResults = { - get: rejectedPromise, - set: rejectedPromise, - delete: rejectedPromise, - getKeysByPrefix: rejectedPromise, - incr: rejectedPromise, - decr: rejectedPromise, - getMany: rejectedPromise, - connect: rejectedPromise, - close: rejectedPromise -}; diff --git a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts index 0764c182..ea0f2c22 100644 --- a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts +++ b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts @@ -29,7 +29,13 @@ export const wrapperWithIssues = { decr: 'invalid value', getMany: throwsException, connect: rejectedPromise, - close: invalidThenable + close: invalidThenable, + getAndSet: 'invalid value', + getByPrefix: throwsException, + pushItems: rejectedPromise, + popItems: invalidThenable, + getItemsCount: 'invalid value', + itemContains: throwsException, }; export const wrapperWithValuesToSanitize = { @@ -40,7 +46,13 @@ export const wrapperWithValuesToSanitize = { incr: () => Promise.resolve('1'), decr: () => Promise.resolve('0'), getMany: () => Promise.resolve(['1', null, false, true, '2', null]), - connect: () => Promise.resolve(1) + connect: () => Promise.resolve(1), + getAndSet: () => Promise.resolve(true), + getByPrefix: () => Promise.resolve(['1', null, false, true, '2', null]), + pushItems: () => Promise.resolve('false'), + popItems: () => Promise.resolve('invalid array'), + getItemsCount: () => Promise.resolve('10'), + itemContains: () => Promise.resolve('true'), }; const SANITIZED_RESULTS = { @@ -51,7 +63,13 @@ const SANITIZED_RESULTS = { incr: false, decr: false, getMany: ['1', null, '2', null], - connect: false + connect: false, + getAndSet: 'true', + getByPrefix: ['1', '2'], + pushItems: false, + popItems: [], + getItemsCount: 10, + itemContains: true, }; const VALID_METHOD_CALLS = { @@ -63,7 +81,13 @@ const VALID_METHOD_CALLS = { 'decr': ['some_key'], 'getMany': [['some_key_1', 'some_key_2']], 'connect': [], - 'close': [] + 'close': [], + 'getAndSet': ['some_key', 'some_value'], + 'getByPrefix': ['some_prefix'], + 'pushItems': ['some_key', ['item1', 'item2']], + 'popItems': ['some_key'], + 'getItemsCount': ['some_key'], + 'itemContains': ['some_key', 'some_value'], }; // Test target @@ -125,7 +149,7 @@ describe('Wrapper Adapter', () => { expect(result).toEqual(SANITIZED_RESULTS[method]); // result should be sanitized } catch (e) { - expect(true).toBe(false); // promise shouldn't be rejected + expect(true).toBe(methods[i]); // promise shouldn't be rejected } } diff --git a/src/storages/pluggable/wrapperAdapter.ts b/src/storages/pluggable/wrapperAdapter.ts index a442b254..37fa666d 100644 --- a/src/storages/pluggable/wrapperAdapter.ts +++ b/src/storages/pluggable/wrapperAdapter.ts @@ -1,4 +1,4 @@ -import { toString, isString } from '../../utils/lang'; +import { toString, isString, toNumber } from '../../utils/lang'; import { sanitizeBoolean as sBoolean } from '../../evaluator/value/sanitize'; import { ILogger } from '../../logger/types'; import { SplitError } from '../../utils/lang/errors'; @@ -12,6 +12,11 @@ function sanitizeBoolean(val: any): boolean { } sanitizeBoolean.type = 'boolean'; +function sanitizeNumber(val: any): number { + return toNumber(val); +} +sanitizeNumber.type = 'number'; + function sanitizeArray(val: any): string[] { if (!Array.isArray(val)) return []; // if not an array, returns a new empty one if (val.every(isString)) return val; // if all items are valid, return the given array @@ -42,7 +47,13 @@ const METHODS_TO_PROMISE_WRAP: [string, undefined | { (val: any): any, type: str ['decr', sanitizeBoolean], ['getMany', sanitizeArrayOfNullableString], ['connect', sanitizeBoolean], - ['close', undefined] + ['close', undefined], + ['getAndSet', sanitizeNullableString], + ['getByPrefix', sanitizeArray], + ['pushItems', sanitizeBoolean], + ['popItems', sanitizeArray], + ['getItemsCount', sanitizeNumber], + ['itemContains', sanitizeBoolean], ]; /** diff --git a/src/storages/types.ts b/src/storages/types.ts index 5fe8c1e1..f7f23eda 100644 --- a/src/storages/types.ts +++ b/src/storages/types.ts @@ -8,27 +8,38 @@ import { SplitIO, ImpressionDTO } from '../types'; */ export interface ICustomStorageWrapper { /** - * Returns a promise that resolves with the element associated with the specified `key`, or null if the key can't be found in the storage. + * Return a promise that resolves with the element value associated with the specified `key`, or null if the key can't be found in the storage. */ get: (key: string) => Promise /** - * Adds or updates an element with a specified `key` and a `value`. + * Add or update an element with a specified `key` and `value`. * Returns a promise that resolves with a boolean value: * - true if the element existed and was updated, * - or false if the element didn't exist and was added. */ set: (key: string, value: string) => Promise /** - * Removes the specified element by `key`. + * Add or update an element with a specified `key` and `value`. + * Returns a promise that resolves with the previous value associated to the given `key`, or null if not set. + * - true if the element existed and was updated, + * - or false if the element didn't exist and was added. + */ + getAndSet: (key: string, value: string) => Promise + /** + * Remove the specified element by `key`. * Return a promise that resolves with a boolean value: * - true if the element existed and has been removed, * - or false if the element does not exist. */ del: (key: string) => Promise /** - * Returns a promise that resolves with the list of element keys that match with the given `prefix`. + * Return a promise that resolves with the list of element keys that match with the given `prefix`. */ getKeysByPrefix: (prefix: string) => Promise + /** + * Return a promise that resolves with the list of element values that match with the given `prefix`. + */ + getByPrefix: (prefix: string) => Promise /** * Increment in 1 the given `key` value or set it in 1 if the value doesn't exist. * Return a resolved promise with a boolean value: @@ -46,9 +57,26 @@ export interface ICustomStorageWrapper { */ decr: (key: string) => Promise /** - * Returns a promise that resolves with the list of elements associated with the specified list of `keys`. + * Return a promise that resolves with the list of elements associated with the specified list of `keys`. */ getMany: (keys: string[]) => Promise<(string | null)[]> + /** + * Push given `items` to `key` queue. + */ + pushItems: (key: string, items: string[]) => Promise + /** + * Pop `count` number of items from `key` queue. + * Return a promise that resolves with the list of removed items the removed members, or an empty array when key does not exist. + */ + popItems: (key: string, count: number) => Promise + /** + * Return a promise that resolves with the number of items at the `key` queue, or 0 when key does not exist. + */ + getItemsCount: (key: string) => Promise + /** + * Return a promise that resolves with true boolean value if `item` is a member of the set stored at `key`, or false otherwise. + */ + itemContains: (key: string, item: string) => Promise /** * For storages that requires to be connected, like database servers. From 395d3124db6aaad688af55cb07dcb10d19c84a84 Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Wed, 28 Apr 2021 09:32:23 -0300 Subject: [PATCH 07/14] implementation and unit tests --- .../pluggable/SegmentsCachePluggable.ts | 93 +++++++++++++++++++ .../__tests__/SegmentsCachePluggable.spec.ts | 36 +++++++ .../pluggable/__tests__/wrapper.mock.ts | 7 ++ 3 files changed, 136 insertions(+) create mode 100644 src/storages/pluggable/SegmentsCachePluggable.ts create mode 100644 src/storages/pluggable/__tests__/SegmentsCachePluggable.spec.ts diff --git a/src/storages/pluggable/SegmentsCachePluggable.ts b/src/storages/pluggable/SegmentsCachePluggable.ts new file mode 100644 index 00000000..67fc39be --- /dev/null +++ b/src/storages/pluggable/SegmentsCachePluggable.ts @@ -0,0 +1,93 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +/* eslint-disable no-unused-vars */ +import { isNaNNumber } from '../../utils/lang'; +import KeyBuilder from '../KeyBuilder'; +import { ICustomStorageWrapper, ISegmentsCacheAsync } from '../types'; +import { ILogger } from '../../logger/types'; +import { logPrefix } from './constants'; + +/** + * ISegmentsCacheAsync implementation for pluggable storages. + */ +export class SegmentsCachePluggable implements ISegmentsCacheAsync { + + private readonly log: ILogger; + private readonly keys: KeyBuilder; + private readonly wrapper: ICustomStorageWrapper; + + constructor(log: ILogger, keys: KeyBuilder, wrapper: ICustomStorageWrapper) { + this.log = log; + this.keys = keys; + this.wrapper = wrapper; + } + + /** + * Returns if `key` is part of `name` segment. + */ + isInSegment(name: string, key: string) { + return this.wrapper.itemContains(this.keys.buildSegmentNameKey(name), key); + } + + /** + * Set till number for the given segment `name`. + * The returned promise is resolved when the operation success, + * or rejected with an SplitError if it fails (e.g., wrapper operation fails). + */ + setChangeNumber(name: string, changeNumber: number) { + return this.wrapper.set( + this.keys.buildSegmentTillKey(name), changeNumber + '' + ); + } + + /** + * Get till number or -1 if it's not defined. + * The returned promise is resolved with the changeNumber or -1 if it doesn't exist or a wrapper operation fails. + * The promise will never be rejected. + */ + getChangeNumber(name: string) { + return this.wrapper.get(this.keys.buildSegmentTillKey(name)).then((value: string | null) => { + const i = parseInt(value as string, 10); + return isNaNNumber(i) ? -1 : i; + }).catch((e) => { + this.log.error(logPrefix + 'Could not retrieve changeNumber from segments storage. Error: ' + e); + return -1; + }); + } + + /** + * Add a list of `segmentKeys` to the given segment `name`. + * The returned promise is resolved when the operation success + * or rejected with an SplitError if it fails (e.g., wrapper operation fails, JSON parsing error) + * + * @TODO implement for DataLoader/Producer mode + */ + addToSegment(name: string, segmentKeys: string[]) { + return Promise.resolve(true); + } + + /** + * Remove a list of `segmentKeys` from the given segment `name`. + * The returned promise is resolved when the operation success + * or rejected with an SplitError if it fails (e.g., wrapper operation fails, JSON parsing error) + * + * @TODO implement for DataLoader/Producer mode + */ + removeFromSegment(name: string, segmentKeys: string[]) { + return Promise.resolve(true); + } + + /** @TODO implement for DataLoader/Producer mode */ + registerSegments(names: string[]) { + return Promise.resolve(true); + } + + /** @TODO implement for DataLoader/Producer mode */ + getRegisteredSegments() { + return Promise.resolve([]); + } + + /** @TODO implement for DataLoader/Producer mode */ + clear(): Promise { + return Promise.resolve(true); + } +} diff --git a/src/storages/pluggable/__tests__/SegmentsCachePluggable.spec.ts b/src/storages/pluggable/__tests__/SegmentsCachePluggable.spec.ts new file mode 100644 index 00000000..4610b00a --- /dev/null +++ b/src/storages/pluggable/__tests__/SegmentsCachePluggable.spec.ts @@ -0,0 +1,36 @@ +import { SegmentsCachePluggable } from '../SegmentsCachePluggable'; +import KeyBuilder from '../../KeyBuilder'; +import { loggerMock } from '../../../logger/__tests__/sdkLogger.mock'; +import { wrapperMock } from './wrapper.mock'; + +const keyBuilder = new KeyBuilder(); + +describe('SEGMENTS CACHE PLUGGABLE', () => { + + afterEach(() => { + loggerMock.mockClear(); + wrapperMock.mockClear(); + }); + + test('isInSegment, set/getChangeNumber', async () => { + const cache = new SegmentsCachePluggable(loggerMock, keyBuilder, wrapperMock); + + // assert setChangeNumber and getChangeNumber + expect(await cache.setChangeNumber('mocked-segment', 100)).toBe(false); + expect(await cache.getChangeNumber('mocked-segment')).toBe(100); + expect(await cache.setChangeNumber('mocked-segment', 200)).toBe(true); + expect(await cache.getChangeNumber('mocked-segment')).toBe(200); + expect(await cache.getChangeNumber('inexistent-segment')).toBe(-1); // -1 if the segment doesn't exist + + // mock segment keys + wrapperMock._queues[keyBuilder.buildSegmentNameKey('mocked-segment')] = ['b', 'd']; + + expect(await cache.isInSegment('mocked-segment', 'a')).toBe(false); + expect(await cache.isInSegment('mocked-segment', 'b')).toBe(true); + expect(await cache.isInSegment('mocked-segment', 'c')).toBe(false); + expect(await cache.isInSegment('mocked-segment', 'd')).toBe(true); + + expect(await cache.isInSegment('inexistent-segment', 'a')).toBe(false); + }); + +}); diff --git a/src/storages/pluggable/__tests__/wrapper.mock.ts b/src/storages/pluggable/__tests__/wrapper.mock.ts index 1f8af875..7be7f9fe 100644 --- a/src/storages/pluggable/__tests__/wrapper.mock.ts +++ b/src/storages/pluggable/__tests__/wrapper.mock.ts @@ -77,6 +77,7 @@ export const wrapperMock = { mockClear() { _cache = {}; + _queues = {}; this.get.mockClear(); this.set.mockClear(); this.del.mockClear(); @@ -86,5 +87,11 @@ export const wrapperMock = { this.getMany.mockClear(); this.connect.mockClear(); this.close.mockClear(); + this.getAndSet.mockClear(); + this.getByPrefix.mockClear(); + this.pushItems.mockClear(); + this.popItems.mockClear(); + this.getItemsCount.mockClear(); + this.itemContains.mockClear(); } }; From 5cee3ba2c39261b4178901c23b57f21e8cede9c0 Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Wed, 28 Apr 2021 14:33:22 -0300 Subject: [PATCH 08/14] applied Emi's feedback --- src/storages/inRedis/SplitsCacheInRedis.ts | 40 +++++++++---------- .../pluggable/SplitsCachePluggable.ts | 14 +++---- .../__tests__/splitExistance.spec.ts | 2 +- .../__tests__/trafficTypeExistance.spec.ts | 8 ++-- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/storages/inRedis/SplitsCacheInRedis.ts b/src/storages/inRedis/SplitsCacheInRedis.ts index c9694862..726ec0a6 100644 --- a/src/storages/inRedis/SplitsCacheInRedis.ts +++ b/src/storages/inRedis/SplitsCacheInRedis.ts @@ -23,21 +23,23 @@ function processPipelineAnswer(results: Array<[Error | null, string]>): string[] */ export default class SplitsCacheInRedis implements ISplitsCacheAsync { + private readonly log: ILogger; private readonly redis: Redis; private readonly keys: KeyBuilderSS; - // private redisError?: string; + private redisError?: string; - constructor(private readonly log: ILogger, keys: KeyBuilderSS, redis: Redis) { + constructor(log: ILogger, keys: KeyBuilderSS, redis: Redis) { + this.log = log; this.redis = redis; this.keys = keys; - // this.redis.on('error', (e) => { - // this.redisError = e; - // }); + this.redis.on('error', (e) => { + this.redisError = e; + }); - // this.redis.on('connect', () => { - // this.redisError = undefined; - // }); + this.redis.on('connect', () => { + this.redisError = undefined; + }); } // @TODO fix: incr/decr TT and segments for producer mode. Follow pluggable storage signature @@ -91,12 +93,11 @@ export default class SplitsCacheInRedis implements ISplitsCacheAsync { * Returned promise is Rejected with an SplitError if redis operation fails. */ getSplit(name: string): Promise { - // @TODO remove next block if not required - // if (this.redisError) { - // this.log.error(logPrefix + this.redisError); + if (this.redisError) { + this.log.error(logPrefix + this.redisError); - // throw this.redisError; - // } + return Promise.reject(this.redisError); // no need to wrap as an SplitError + } return this.redis.get(this.keys.buildSplitKey(name)); } @@ -149,14 +150,14 @@ export default class SplitsCacheInRedis implements ISplitsCacheAsync { ttCount = parseInt(ttCount as string, 10); if (!isFiniteNumber(ttCount) || ttCount < 0) { - this.log.info(logPrefix + `Could not validate traffic type existance of ${trafficType} due to data corruption of some sorts.`); + this.log.info(logPrefix + `Could not validate traffic type existence of ${trafficType} due to data corruption of some sorts.`); return false; } return ttCount > 0; }) .catch(e => { - this.log.error(logPrefix + `Could not validate traffic type existance of ${trafficType} due to an error: ${e}.`); + this.log.error(logPrefix + `Could not validate traffic type existence of ${trafficType} due to an error: ${e}.`); // If there is an error, bypass the validation so the event can get tracked. return true; }); @@ -181,12 +182,11 @@ export default class SplitsCacheInRedis implements ISplitsCacheAsync { * Returned promise is Rejected with an SplitError if redis operation fails. */ getSplits(names: string[]): Promise> { - // @TODO remove next block if not required - // if (this.redisError) { - // this.log.error(logPrefix + this.redisError); + if (this.redisError) { + this.log.error(logPrefix + this.redisError); - // throw this.redisError; - // } + return Promise.reject(this.redisError); // no need to wrap as an SplitError + } const splits: Record = {}; const keys = names.map(name => this.keys.buildSplitKey(name)); diff --git a/src/storages/pluggable/SplitsCachePluggable.ts b/src/storages/pluggable/SplitsCachePluggable.ts index 68e9486b..cc035cea 100644 --- a/src/storages/pluggable/SplitsCachePluggable.ts +++ b/src/storages/pluggable/SplitsCachePluggable.ts @@ -17,10 +17,10 @@ export class SplitsCachePluggable implements ISplitsCacheAsync { private readonly wrapper: ICustomStorageWrapper; /** - * - * @param log logger instance - * @param keys key builder - * @param wrapper adapted wrapper storage + * Create a SplitsCache that uses a custom storage wrapper. + * @param log Logger instance. + * @param keys Key builder. + * @param wrapper Adapted wrapper storage. */ constructor(log: ILogger, keys: KeyBuilder, wrapper: ICustomStorageWrapper) { this.log = log; @@ -167,7 +167,7 @@ export class SplitsCachePluggable implements ISplitsCacheAsync { } /** - * Check traffic type existance. + * Check traffic type existence. * The returned promise is resolved with a boolean indicating whether the TT exist or not. * In case of wrapper operation failures, the promise resolves with a true value, assuming that the TT might exist. * It will never be rejected. @@ -180,13 +180,13 @@ export class SplitsCachePluggable implements ISplitsCacheAsync { ttCount = parseInt(ttCount as string, 10); if (!isFiniteNumber(ttCount) || ttCount < 0) { - this.log.info(logPrefix + `Could not validate traffic type existance of ${trafficType} due to data corruption of some sorts.`); + this.log.info(logPrefix + `Could not validate traffic type existence of ${trafficType} due to data corruption of some sorts.`); return false; } return ttCount > 0; }).catch(e => { - this.log.error(logPrefix + `Could not validate traffic type existance of ${trafficType} due to an error: ${e}.`); + this.log.error(logPrefix + `Could not validate traffic type existence of ${trafficType} due to an error: ${e}.`); // If there is an error, bypass the validation so the event can get tracked. return true; }); diff --git a/src/utils/inputValidation/__tests__/splitExistance.spec.ts b/src/utils/inputValidation/__tests__/splitExistance.spec.ts index 4bc2b8e8..7d0d7b39 100644 --- a/src/utils/inputValidation/__tests__/splitExistance.spec.ts +++ b/src/utils/inputValidation/__tests__/splitExistance.spec.ts @@ -7,7 +7,7 @@ import { validateSplitExistance } from '../splitExistance'; import { IReadinessManager } from '../../../readiness/types'; import { WARN_NOT_EXISTENT_SPLIT } from '../../../logger/constants'; -describe('Split existance (special case)', () => { +describe('Split existence (special case)', () => { afterEach(() => { loggerMock.mockClear(); }); diff --git a/src/utils/inputValidation/__tests__/trafficTypeExistance.spec.ts b/src/utils/inputValidation/__tests__/trafficTypeExistance.spec.ts index ba2f1666..293ccb84 100644 --- a/src/utils/inputValidation/__tests__/trafficTypeExistance.spec.ts +++ b/src/utils/inputValidation/__tests__/trafficTypeExistance.spec.ts @@ -57,7 +57,7 @@ describe('validateTrafficTypeExistance', () => { readinessManagerMock.isReady.mockImplementation(() => true); expect(validateTrafficTypeExistance(loggerMock, readinessManagerMock, splitsCacheMock, STANDALONE_MODE, TEST_EXISTENT_TT, 'test_method')).toBe(true); // If the SDK is in condition to validate but the TT exists, it will return true. - expect(splitsCacheMock.trafficTypeExists.mock.calls).toEqual([[TEST_EXISTENT_TT]]); // If the SDK is in condition to validate, it checks that TT existance with the storage. + expect(splitsCacheMock.trafficTypeExists.mock.calls).toEqual([[TEST_EXISTENT_TT]]); // If the SDK is in condition to validate, it checks that TT existence with the storage. expect(loggerMock.warn).not.toBeCalled(); // If the SDK is in condition to validate but the TT exists, it will not log any warnings. expect(loggerMock.error).not.toBeCalled(); // If the SDK is in condition to validate but the TT exists, it will not log any errors. }); @@ -65,7 +65,7 @@ describe('validateTrafficTypeExistance', () => { test('Should return false and log warning if SDK Ready, not localhost mode and the traffic type does NOT exist in the storage', () => { // Ready, standalone, and the TT not exists in the storage. expect(validateTrafficTypeExistance(loggerMock, readinessManagerMock, splitsCacheMock, STANDALONE_MODE, TEST_NOT_EXISTENT_TT, 'test_method_y')).toBe(false); // If the SDK is in condition to validate but the TT does not exist in the storage, it will return false. - expect(splitsCacheMock.trafficTypeExists.mock.calls).toEqual([[TEST_NOT_EXISTENT_TT]]); // If the SDK is in condition to validate, it checks that TT existance with the storage. + expect(splitsCacheMock.trafficTypeExists.mock.calls).toEqual([[TEST_NOT_EXISTENT_TT]]); // If the SDK is in condition to validate, it checks that TT existence with the storage. expect(loggerMock.warn).toBeCalledWith(WARN_NOT_EXISTENT_TT, ['test_method_y', TEST_NOT_EXISTENT_TT]); // If the SDK is in condition to validate but the TT does not exist in the storage, it will log the expected warning. expect(loggerMock.error).not.toBeCalled(); // It logged a warning so no errors should be logged. }); @@ -75,7 +75,7 @@ describe('validateTrafficTypeExistance', () => { const validationPromise = validateTrafficTypeExistance(loggerMock, readinessManagerMock, splitsCacheMock, STANDALONE_MODE, TEST_EXISTENT_ASYNC_TT, 'test_method_z'); expect(thenable(validationPromise)).toBe(true); // If the storage is async, it should also return a promise. - expect(splitsCacheMock.trafficTypeExists.mock.calls).toEqual([[TEST_EXISTENT_ASYNC_TT]]); // If the SDK is in condition to validate, it checks that TT existance with the async storage. + expect(splitsCacheMock.trafficTypeExists.mock.calls).toEqual([[TEST_EXISTENT_ASYNC_TT]]); // If the SDK is in condition to validate, it checks that TT existence with the async storage. expect(loggerMock.warn).not.toBeCalled(); // We are still fetching the data from the storage, no logs yet. expect(loggerMock.error).not.toBeCalled(); // We are still fetching the data from the storage, no logs yet. @@ -90,7 +90,7 @@ describe('validateTrafficTypeExistance', () => { const validationPromise2 = validateTrafficTypeExistance(loggerMock, readinessManagerMock, splitsCacheMock, STANDALONE_MODE, TEST_NOT_EXISTENT_ASYNC_TT, 'test_method_z'); expect(thenable(validationPromise2)).toBe(true); // If the storage is async, it should also return a promise. - expect(splitsCacheMock.trafficTypeExists.mock.calls).toEqual([[TEST_NOT_EXISTENT_ASYNC_TT]]); // If the SDK is in condition to validate, it checks that TT existance with the async storage. + expect(splitsCacheMock.trafficTypeExists.mock.calls).toEqual([[TEST_NOT_EXISTENT_ASYNC_TT]]); // If the SDK is in condition to validate, it checks that TT existence with the async storage. expect(loggerMock.warn).not.toBeCalled(); // We are still fetching the data from the storage, no logs yet. expect(loggerMock.error).not.toBeCalled(); // We are still fetching the data from the storage, no logs yet. From e487f86c6eee31fded054098867ba6a8419846b1 Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Wed, 28 Apr 2021 18:47:13 -0300 Subject: [PATCH 09/14] implementation and unit tests --- .../pluggable/EventsCachePluggable.ts | 45 +++++ .../pluggable/ImpressionsCachePluggable.ts | 61 ++++++ .../__tests__/EventsCachePluggable.spec.ts | 77 ++++++++ .../ImpressionsCachePluggable.spec.ts | 84 ++++++++ .../pluggable/__tests__/wrapper.mock.ts | 184 +++++++++--------- .../__tests__/wrapperAdapter.spec.ts | 2 +- 6 files changed, 362 insertions(+), 91 deletions(-) create mode 100644 src/storages/pluggable/EventsCachePluggable.ts create mode 100644 src/storages/pluggable/ImpressionsCachePluggable.ts create mode 100644 src/storages/pluggable/__tests__/EventsCachePluggable.spec.ts create mode 100644 src/storages/pluggable/__tests__/ImpressionsCachePluggable.spec.ts diff --git a/src/storages/pluggable/EventsCachePluggable.ts b/src/storages/pluggable/EventsCachePluggable.ts new file mode 100644 index 00000000..eef10dd1 --- /dev/null +++ b/src/storages/pluggable/EventsCachePluggable.ts @@ -0,0 +1,45 @@ +import { ICustomStorageWrapper, IEventsCacheAsync } from '../types'; +import { IRedisMetadata } from '../../dtos/types'; +import KeyBuilderSS from '../KeyBuilderSS'; +import { SplitIO } from '../../types'; +import { ILogger } from '../../logger/types'; +import { logPrefix } from './constants'; + +export class EventsCachePluggable implements IEventsCacheAsync { + + private readonly log: ILogger; + private readonly wrapper: ICustomStorageWrapper; + private readonly keys: KeyBuilderSS; + private readonly metadata: IRedisMetadata; + + constructor(log: ILogger, keys: KeyBuilderSS, wrapper: ICustomStorageWrapper, metadata: IRedisMetadata) { + this.log = log; + this.keys = keys; + this.wrapper = wrapper; + this.metadata = metadata; + } + + /** + * Push given event to the storage. + * @param eventData Event item to push. + * @returns A promise that is resolved with a boolean value indicating if the push operation succeeded or failed. + * The promise will never be rejected. + */ + track(eventData: SplitIO.EventData): Promise { + return this.wrapper.pushItems( + this.keys.buildEventsKey(), + [this._toJSON(eventData)] + ).catch(e => { + this.log.error(logPrefix + ` Error adding event to queue: ${e}.`); + return false; + }); + } + + private _toJSON(eventData: SplitIO.EventData): string { + return JSON.stringify({ + m: this.metadata, + e: eventData + }); + } + +} diff --git a/src/storages/pluggable/ImpressionsCachePluggable.ts b/src/storages/pluggable/ImpressionsCachePluggable.ts new file mode 100644 index 00000000..643b64de --- /dev/null +++ b/src/storages/pluggable/ImpressionsCachePluggable.ts @@ -0,0 +1,61 @@ +import { ICustomStorageWrapper, IImpressionsCacheAsync } from '../types'; +import { IRedisMetadata } from '../../dtos/types'; +import { ImpressionDTO } from '../../types'; +import KeyBuilderSS from '../KeyBuilderSS'; +import { ILogger } from '../../logger/types'; +import { logPrefix } from './constants'; + +export class ImpressionsCachePluggable implements IImpressionsCacheAsync { + + private readonly log: ILogger; + private readonly keys: KeyBuilderSS; + private readonly wrapper: ICustomStorageWrapper; + private readonly metadata: IRedisMetadata; + + constructor(log: ILogger, keys: KeyBuilderSS, wrapper: ICustomStorageWrapper, metadata: IRedisMetadata) { + this.log = log; + this.keys = keys; + this.wrapper = wrapper; + this.metadata = metadata; + } + + /** + * Push given impressions to the storage. + * @param impressions List of impresions to push. + * @returns A promise that is resolved with a boolean value indicating if the push operation succeeded or failed. + * The promise will never be rejected. + */ + track(impressions: ImpressionDTO[]): Promise { + return this.wrapper.pushItems( + this.keys.buildImpressionsKey(), + this._toJSON(impressions) + ).catch((e) => { + this.log.error(logPrefix + ` Error adding event to queue: ${e}.`); + return false; + }); + } + + private _toJSON(impressions: ImpressionDTO[]): string[] { + return impressions.map(impression => { + const { + keyName, bucketingKey, feature, treatment, label, time, changeNumber + } = impression; + + return JSON.stringify({ + m: this.metadata, + i: { + k: keyName, + b: bucketingKey, + f: feature, + t: treatment, + r: label, + c: changeNumber, + m: time + } + }); + }); + } + + // @TODO implement producer methods + +} diff --git a/src/storages/pluggable/__tests__/EventsCachePluggable.spec.ts b/src/storages/pluggable/__tests__/EventsCachePluggable.spec.ts new file mode 100644 index 00000000..e7f168fb --- /dev/null +++ b/src/storages/pluggable/__tests__/EventsCachePluggable.spec.ts @@ -0,0 +1,77 @@ +import find from 'lodash/find'; +import isEqual from 'lodash/isEqual'; +import { loggerMock } from '../../../logger/__tests__/sdkLogger.mock'; +import KeyBuilderSS from '../../KeyBuilderSS'; +import { EventsCachePluggable } from '../EventsCachePluggable'; +import { metadataBuilder } from '../../inRedis'; +import { wrapperMockFactory } from './wrapper.mock'; +import { wrapperAdapter } from '../wrapperAdapter'; + +const prefix = 'events_cache_ut'; +const metadata = { version: 'js_someversion', ip: 'some_ip', hostname: 'some_hostname' }; +const keys = new KeyBuilderSS(prefix, metadata); +const key = keys.buildEventsKey(); + +const fakeRedisMetadata = metadataBuilder(metadata); +const fakeEvent1 = { event: 1 }; +const fakeEvent2 = { event: '2' }; +const fakeEvent3 = { event: null }; + +describe('PLUGGABLE EVENTS CACHE', () => { + + test('`track` method should push values into the pluggable storage', async () => { + const wrapperMock = wrapperMockFactory(); + const cache = new EventsCachePluggable(loggerMock, keys, wrapperMock, fakeRedisMetadata); + + // @ts-expect-error + expect(await cache.track(fakeEvent1)).toBe(true); // If the queueing operation was successful, it should resolve the returned promise with "true" + // @ts-expect-error + expect(await cache.track(fakeEvent2)).toBe(true); // If the queueing operation was successful, it should resolve the returned promise with "true" + // @ts-expect-error + expect(await cache.track(fakeEvent3)).toBe(true); // If the queueing operation was successful, it should resolve the returned promise with "true" + + const redisValues = wrapperMock._queues[key]; + + expect(redisValues.length).toBe(3); // After pushing we should have on Redis as many events as we have stored. + expect(typeof redisValues[0]).toBe('string'); // All elements should be strings since those are stringified JSONs. + expect(typeof redisValues[1]).toBe('string'); // All elements should be strings since those are stringified JSONs. + expect(typeof redisValues[2]).toBe('string'); // All elements should be strings since those are stringified JSONs. + + const findMatchingElem = (event: any) => { + return find(redisValues, elem => { + const parsedElem = JSON.parse(elem); + return isEqual(parsedElem.e, event) && isEqual(parsedElem.m, fakeRedisMetadata); + }); + }; + + /* If the elements are found, then the values are correct. */ + const foundEv1 = findMatchingElem(fakeEvent1); + const foundEv2 = findMatchingElem(fakeEvent2); + const foundEv3 = findMatchingElem(fakeEvent3); + expect(foundEv1).not.toBe(undefined); // Events stored on redis matched the values we are expecting. + expect(foundEv2).not.toBe(undefined); // Events stored on redis matched the values we are expecting. + expect(foundEv3).not.toBe(undefined); // Events stored on redis matched the values we are expecting. + + wrapperMock.mockClear(); + }); + + test('`track` method result should not reject if wrapper operation fails', async () => { + const wrapperMock = wrapperMockFactory(); + // @ts-ignore. I'll use a "bad" queue to force an exception with the pushItems wrapper operation. + wrapperMock._queues['non-list-key'] = 10; + // @ts-expect-error. wrapperMock is adapted this time to properly handle unexpected exceptions + const faultyCache = new EventsCachePluggable(loggerMock, { + buildEventsKey: () => 'non-list-key' + }, wrapperAdapter(loggerMock, wrapperMock), fakeRedisMetadata); + + // @ts-expect-error + expect(await faultyCache.track(fakeEvent1)).toBe(false); // If the queueing operation was NOT successful, it should resolve the returned promise with "false" instead of rejecting it. + // @ts-expect-error + expect(await faultyCache.track(fakeEvent2)).toBe(false); // If the queueing operation was NOT successful, it should resolve the returned promise with "false" instead of rejecting it. + // @ts-expect-error + expect(await faultyCache.track(fakeEvent3)).toBe(false); // If the queueing operation was NOT successful, it should resolve the returned promise with "false" instead of rejecting it. + + wrapperMock.mockClear(); + }); + +}); diff --git a/src/storages/pluggable/__tests__/ImpressionsCachePluggable.spec.ts b/src/storages/pluggable/__tests__/ImpressionsCachePluggable.spec.ts new file mode 100644 index 00000000..42dbeedb --- /dev/null +++ b/src/storages/pluggable/__tests__/ImpressionsCachePluggable.spec.ts @@ -0,0 +1,84 @@ + +import KeyBuilderSS from '../../KeyBuilderSS'; +import { ImpressionsCachePluggable } from '../ImpressionsCachePluggable'; +import { loggerMock } from '../../../logger/__tests__/sdkLogger.mock'; +import { wrapperMock } from './wrapper.mock'; +import { IRedisMetadata } from '../../../dtos/types'; + +const prefix = 'impr_cache_ut'; +const impressionsKey = `${prefix}.impressions`; +const testMeta: IRedisMetadata = { i: 'some_ip', n: 'some_host', s: 'some_sdk_version' }; // @ts-ignore +const keys = new KeyBuilderSS(prefix, testMeta); + +const o1 = { + feature: 'test1', + keyName: 'facundo@split.io', + treatment: 'on', + time: Date.now(), + label: 'default rule', + changeNumber: 1 +}; + +const o2 = { + feature: 'test2', + keyName: 'pepep@split.io', + treatment: 'A', + time: Date.now(), + bucketingKey: '1234-5678', + label: 'is in segment', + changeNumber: 1 +}; + +const o3 = { + feature: 'test3', + keyName: 'pipiip@split.io', + treatment: 'B', + time: Date.now(), + label: 'default rule', + changeNumber: 1 +}; + +describe('PLUGGABLE IMPRESSIONS CACHE', () => { + + afterEach(() => { + loggerMock.mockClear(); + wrapperMock.mockClear(); + }); + + test('`track` method should push values into the pluggable storage', async () => { + const cache = new ImpressionsCachePluggable(loggerMock, keys, wrapperMock, testMeta); + + expect(await cache.track([o1])).toBe(true); // result should resolve to true + + let state = wrapperMock._queues[impressionsKey]; + expect(state.length).toBe(1); // impression should be stored + + expect(await cache.track([o2, o3])).toBe(true); + + state = wrapperMock._queues[impressionsKey]; + expect(state.length).toBe(3); + // This is testing both the track and the toJSON method. + expect(state[0]).toBe(JSON.stringify({ + m: testMeta, + i: { k: o1.keyName, f: o1.feature, t: o1.treatment, r: o1.label, c: o1.changeNumber, m: o1.time } + })); + expect(state[1]).toBe(JSON.stringify({ + m: testMeta, + i: { k: o2.keyName, b: o2.bucketingKey, f: o2.feature, t: o2.treatment, r: o2.label, c: o2.changeNumber, m: o2.time } + })); + expect(state[2]).toBe(JSON.stringify({ + m: testMeta, + i: { k: o3.keyName, f: o3.feature, t: o3.treatment, r: o3.label, c: o3.changeNumber, m: o3.time } + })); + + }); + + test('`track` method result should not reject if wrapper operation fails', async () => { + // make wrapper operation fail + wrapperMock.pushItems.mockImplementation(()=>Promise.reject()); + const cache = new ImpressionsCachePluggable(loggerMock, keys, wrapperMock, testMeta); + + expect(await cache.track([o1])).toBe(false); // result should resolve to false + }); + +}); diff --git a/src/storages/pluggable/__tests__/wrapper.mock.ts b/src/storages/pluggable/__tests__/wrapper.mock.ts index 7be7f9fe..352c67cb 100644 --- a/src/storages/pluggable/__tests__/wrapper.mock.ts +++ b/src/storages/pluggable/__tests__/wrapper.mock.ts @@ -1,97 +1,101 @@ import { startsWith, toNumber } from '../../../utils/lang'; -let _cache: Record = {}; -let _queues: Record = {}; +// Creates an in memory ICustomStorageWrapper implementation with Jest mocks +export function wrapperMockFactory() { -// An in memory ICustomStorageWrapper implementation with Jest mocks -export const wrapperMock = { + let _cache: Record = {}; + let _queues: Record = {}; - _cache, - _queues, + return { + _cache, + _queues, - get: jest.fn((key => { - return Promise.resolve(key in _cache ? _cache[key] : null); - })), - set: jest.fn((key: string, value: string) => { - const result = key in _cache; - _cache[key] = value; - return Promise.resolve(result); - }), - getAndSet: jest.fn((key: string, value: string) => { - const result = key in _cache ? _cache[key] : null; - _cache[key] = value; - return Promise.resolve(result); - }), - del: jest.fn((key: string) => { - const result = key in _cache; - delete _cache[key]; - return Promise.resolve(result); - }), - getKeysByPrefix: jest.fn((prefix: string) => { - return Promise.resolve(Object.keys(_cache).filter(key => startsWith(key, prefix))); - }), - getByPrefix: jest.fn((prefix: string) => { - return Promise.resolve(Object.keys(_cache).filter(key => startsWith(key, prefix)).map(key => _cache[key])); - }), - incr: jest.fn((key: string) => { - if (key in _cache) { - const count = toNumber(_cache[key]) + 1; - if (isNaN(count)) return Promise.resolve(false); - _cache[key] = count + ''; - } else { - _cache[key] = '1'; - } - return Promise.resolve(true); - }), - decr: jest.fn((key: string) => { - if (key in _cache) { - const count = toNumber(_cache[key]) - 1; - if (isNaN(count)) return Promise.resolve(false); - _cache[key] = count + ''; - } else { - _cache[key] = '-1'; - } - return Promise.resolve(true); - }), - getMany: jest.fn((keys: string[]) => { - return Promise.resolve(keys.map(key => _cache[key] ? _cache[key] : null)); - }), - pushItems: jest.fn((key: string, items: string[]) => { - if (!(key in _queues)) _queues[key] = []; - _queues[key].push(...items); - return Promise.resolve(true); - }), - popItems: jest.fn((key: string, count: number) => { - return Promise.resolve(key in _queues ? _queues[key].splice(0, count) : []); - }), - getItemsCount: jest.fn((key: string) => { - return Promise.resolve(key in _queues ? _queues[key].length : 0); - }), - itemContains: jest.fn((key: string, item: string) => { - return Promise.resolve(key in _queues && _queues[key].indexOf(item) > -1 ? true : false); - }), + get: jest.fn((key => { + return Promise.resolve(key in _cache ? _cache[key] : null); + })), + set: jest.fn((key: string, value: string) => { + const result = key in _cache; + _cache[key] = value; + return Promise.resolve(result); + }), + getAndSet: jest.fn((key: string, value: string) => { + const result = key in _cache ? _cache[key] : null; + _cache[key] = value; + return Promise.resolve(result); + }), + del: jest.fn((key: string) => { + const result = key in _cache; + delete _cache[key]; + return Promise.resolve(result); + }), + getKeysByPrefix: jest.fn((prefix: string) => { + return Promise.resolve(Object.keys(_cache).filter(key => startsWith(key, prefix))); + }), + getByPrefix: jest.fn((prefix: string) => { + return Promise.resolve(Object.keys(_cache).filter(key => startsWith(key, prefix)).map(key => _cache[key])); + }), + incr: jest.fn((key: string) => { + if (key in _cache) { + const count = toNumber(_cache[key]) + 1; + if (isNaN(count)) return Promise.resolve(false); + _cache[key] = count + ''; + } else { + _cache[key] = '1'; + } + return Promise.resolve(true); + }), + decr: jest.fn((key: string) => { + if (key in _cache) { + const count = toNumber(_cache[key]) - 1; + if (isNaN(count)) return Promise.resolve(false); + _cache[key] = count + ''; + } else { + _cache[key] = '-1'; + } + return Promise.resolve(true); + }), + getMany: jest.fn((keys: string[]) => { + return Promise.resolve(keys.map(key => _cache[key] ? _cache[key] : null)); + }), + pushItems: jest.fn((key: string, items: string[]) => { + if (!(key in _queues)) _queues[key] = []; + _queues[key].push(...items); + return Promise.resolve(true); + }), + popItems: jest.fn((key: string, count: number) => { + return Promise.resolve(key in _queues ? _queues[key].splice(0, count) : []); + }), + getItemsCount: jest.fn((key: string) => { + return Promise.resolve(key in _queues ? _queues[key].length : 0); + }), + itemContains: jest.fn((key: string, item: string) => { + return Promise.resolve(key in _queues && _queues[key].indexOf(item) > -1 ? true : false); + }), - // always connects and close - connect: jest.fn(() => Promise.resolve(true)), - close: jest.fn(() => Promise.resolve()), + // always connects and close + connect: jest.fn(() => Promise.resolve(true)), + close: jest.fn(() => Promise.resolve()), + + mockClear() { + this._cache = {}; + this._queues = {}; + this.get.mockClear(); + this.set.mockClear(); + this.del.mockClear(); + this.getKeysByPrefix.mockClear(); + this.incr.mockClear(); + this.decr.mockClear(); + this.getMany.mockClear(); + this.connect.mockClear(); + this.close.mockClear(); + this.getAndSet.mockClear(); + this.getByPrefix.mockClear(); + this.pushItems.mockClear(); + this.popItems.mockClear(); + this.getItemsCount.mockClear(); + this.itemContains.mockClear(); + } + }; +} - mockClear() { - _cache = {}; - _queues = {}; - this.get.mockClear(); - this.set.mockClear(); - this.del.mockClear(); - this.getKeysByPrefix.mockClear(); - this.incr.mockClear(); - this.decr.mockClear(); - this.getMany.mockClear(); - this.connect.mockClear(); - this.close.mockClear(); - this.getAndSet.mockClear(); - this.getByPrefix.mockClear(); - this.pushItems.mockClear(); - this.popItems.mockClear(); - this.getItemsCount.mockClear(); - this.itemContains.mockClear(); - } -}; +export const wrapperMock = wrapperMockFactory(); diff --git a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts index ea0f2c22..c6292f87 100644 --- a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts +++ b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts @@ -6,7 +6,7 @@ import { SplitError } from '../../../utils/lang/errors'; /** Mocks */ -import { wrapperMock } from './wrapper.mock'; // Well implemented wrapper +import { wrapperMock } from './wrapper.mock'; function throwsException() { throw new Error('some error'); From 160944e23e7c3deb7f91a451ef04747aff08c291 Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Thu, 29 Apr 2021 19:59:26 -0300 Subject: [PATCH 10/14] applied feedback --- .../inLocalStorage/MySegmentsCacheInLocal.ts | 8 +++--- .../inLocalStorage/SplitsCacheInLocal.ts | 20 +++++++-------- src/storages/inLocalStorage/constants.ts | 2 +- src/storages/inLocalStorage/index.ts | 4 +-- src/storages/inRedis/EventsCacheInRedis.ts | 5 ++-- src/storages/inRedis/RedisAdapter.ts | 20 +++++++-------- src/storages/inRedis/SplitsCacheInRedis.ts | 13 +++++----- .../inRedis/__tests__/RedisAdapter.spec.ts | 24 +++++++++--------- src/storages/inRedis/constants.ts | 1 + .../pluggable/EventsCachePluggable.ts | 13 ++++++---- .../pluggable/ImpressionsCachePluggable.ts | 13 ++++++---- .../pluggable/SegmentsCachePluggable.ts | 4 +-- .../__tests__/EventsCachePluggable.spec.ts | 20 +++++++-------- .../ImpressionsCachePluggable.spec.ts | 6 ++--- .../pluggable/__tests__/wrapper.mock.ts | 25 +++++++++++-------- .../__tests__/wrapperAdapter.spec.ts | 4 +-- src/storages/pluggable/constants.ts | 2 +- src/storages/pluggable/wrapperAdapter.ts | 10 ++++---- src/storages/types.ts | 14 +++++++---- 19 files changed, 110 insertions(+), 98 deletions(-) create mode 100644 src/storages/inRedis/constants.ts diff --git a/src/storages/inLocalStorage/MySegmentsCacheInLocal.ts b/src/storages/inLocalStorage/MySegmentsCacheInLocal.ts index 020f823a..12289470 100644 --- a/src/storages/inLocalStorage/MySegmentsCacheInLocal.ts +++ b/src/storages/inLocalStorage/MySegmentsCacheInLocal.ts @@ -1,7 +1,7 @@ import { ILogger } from '../../logger/types'; import AbstractSegmentsCacheSync from '../AbstractSegmentsCacheSync'; import KeyBuilderCS from '../KeyBuilderCS'; -import { logPrefix, DEFINED } from './constants'; +import { LOG_PREFIX, DEFINED } from './constants'; export default class MySegmentsCacheInLocal extends AbstractSegmentsCacheSync { @@ -20,7 +20,7 @@ export default class MySegmentsCacheInLocal extends AbstractSegmentsCacheSync { * @NOTE this method is not being used at the moment. */ clear() { - this.log.info(logPrefix + 'Flushing MySegments data from localStorage'); + this.log.info(LOG_PREFIX + 'Flushing MySegments data from localStorage'); // We cannot simply call `localStorage.clear()` since that implies removing user items from the storage // We could optimize next sentence, since it implies iterating over all localStorage items @@ -34,7 +34,7 @@ export default class MySegmentsCacheInLocal extends AbstractSegmentsCacheSync { localStorage.setItem(segmentKey, DEFINED); return true; } catch (e) { - this.log.error(logPrefix + e); + this.log.error(LOG_PREFIX + e); return false; } } @@ -46,7 +46,7 @@ export default class MySegmentsCacheInLocal extends AbstractSegmentsCacheSync { localStorage.removeItem(segmentKey); return true; } catch (e) { - this.log.error(logPrefix + e); + this.log.error(LOG_PREFIX + e); return false; } } diff --git a/src/storages/inLocalStorage/SplitsCacheInLocal.ts b/src/storages/inLocalStorage/SplitsCacheInLocal.ts index 58d5ba09..58f255f7 100644 --- a/src/storages/inLocalStorage/SplitsCacheInLocal.ts +++ b/src/storages/inLocalStorage/SplitsCacheInLocal.ts @@ -3,7 +3,7 @@ import AbstractSplitsCacheSync, { usesSegments } from '../AbstractSplitsCacheSyn import { isFiniteNumber, toNumber, isNaNNumber } from '../../utils/lang'; import KeyBuilderCS from '../KeyBuilderCS'; import { ILogger } from '../../logger/types'; -import { logPrefix } from './constants'; +import { LOG_PREFIX } from './constants'; /** * ISplitsCacheSync implementation that stores split definitions in browser LocalStorage. @@ -52,7 +52,7 @@ export default class SplitsCacheInLocal extends AbstractSplitsCacheSync { } } } catch (e) { - this.log.error(logPrefix + e); + this.log.error(LOG_PREFIX + e); } } @@ -72,7 +72,7 @@ export default class SplitsCacheInLocal extends AbstractSplitsCacheSync { } } } catch (e) { - this.log.error(logPrefix + e); + this.log.error(LOG_PREFIX + e); } } @@ -82,7 +82,7 @@ export default class SplitsCacheInLocal extends AbstractSplitsCacheSync { * We cannot simply call `localStorage.clear()` since that implies removing user items from the storage. */ clear() { - this.log.info(logPrefix + 'Flushing Splits data from localStorage'); + this.log.info(LOG_PREFIX + 'Flushing Splits data from localStorage'); // collect item keys const len = localStorage.length; @@ -114,7 +114,7 @@ export default class SplitsCacheInLocal extends AbstractSplitsCacheSync { return true; } catch (e) { - this.log.error(logPrefix + e); + this.log.error(LOG_PREFIX + e); return false; } } @@ -129,7 +129,7 @@ export default class SplitsCacheInLocal extends AbstractSplitsCacheSync { return 1; } catch (e) { - this.log.error(logPrefix + e); + this.log.error(LOG_PREFIX + e); return 0; } } @@ -147,14 +147,14 @@ export default class SplitsCacheInLocal extends AbstractSplitsCacheSync { // when using a new split query, we must update it at the store if (this.updateNewFilter) { - this.log.info(logPrefix + 'Split filter query was modified. Updating cache.'); + this.log.info(LOG_PREFIX + 'Split filter query was modified. Updating cache.'); const queryKey = this.keys.buildSplitsFilterQueryKey(); const queryString = this.splitFiltersValidation.queryString; try { if (queryString) localStorage.setItem(queryKey, queryString); else localStorage.removeItem(queryKey); } catch (e) { - this.log.error(logPrefix + e); + this.log.error(LOG_PREFIX + e); } this.updateNewFilter = false; } @@ -166,7 +166,7 @@ export default class SplitsCacheInLocal extends AbstractSplitsCacheSync { this.hasSync = true; return true; } catch (e) { - this.log.error(logPrefix + e); + this.log.error(LOG_PREFIX + e); return false; } } @@ -273,7 +273,7 @@ export default class SplitsCacheInLocal extends AbstractSplitsCacheSync { }); } } catch (e) { - this.log.error(logPrefix + e); + this.log.error(LOG_PREFIX + e); } } // if the filter didn't change, nothing is done diff --git a/src/storages/inLocalStorage/constants.ts b/src/storages/inLocalStorage/constants.ts index 74624310..8dcb6c3e 100644 --- a/src/storages/inLocalStorage/constants.ts +++ b/src/storages/inLocalStorage/constants.ts @@ -1,2 +1,2 @@ -export const logPrefix = 'storage:localstorage: '; +export const LOG_PREFIX = 'storage:localstorage: '; export const DEFINED = '1'; diff --git a/src/storages/inLocalStorage/index.ts b/src/storages/inLocalStorage/index.ts index 226d771c..a26587d1 100644 --- a/src/storages/inLocalStorage/index.ts +++ b/src/storages/inLocalStorage/index.ts @@ -10,7 +10,7 @@ import MySegmentsCacheInMemory from '../inMemory/MySegmentsCacheInMemory'; import SplitsCacheInMemory from '../inMemory/SplitsCacheInMemory'; import { DEFAULT_CACHE_EXPIRATION_IN_MILLIS } from '../../utils/constants/browser'; import { InMemoryStorageCSFactory } from '../inMemory/InMemoryStorageCS'; -import { logPrefix } from './constants'; +import { LOG_PREFIX } from './constants'; export interface InLocalStorageOptions { prefix?: string @@ -27,7 +27,7 @@ export function InLocalStorage(options: InLocalStorageOptions = {}) { // Fallback to InMemoryStorage if LocalStorage API is not available if (!isLocalStorageAvailable()) { - params.log.warn(logPrefix + 'LocalStorage API is unavailable. Fallbacking into default MEMORY storage'); + params.log.warn(LOG_PREFIX + 'LocalStorage API is unavailable. Fallbacking into default MEMORY storage'); return InMemoryStorageCSFactory(params); } diff --git a/src/storages/inRedis/EventsCacheInRedis.ts b/src/storages/inRedis/EventsCacheInRedis.ts index 5a8caab5..97f7d13f 100644 --- a/src/storages/inRedis/EventsCacheInRedis.ts +++ b/src/storages/inRedis/EventsCacheInRedis.ts @@ -4,8 +4,7 @@ import KeyBuilderSS from '../KeyBuilderSS'; import { Redis } from 'ioredis'; import { SplitIO } from '../../types'; import { ILogger } from '../../logger/types'; - -const logPrefix = 'storage:redis: '; +import { LOG_PREFIX } from './constants'; export default class EventsCacheInRedis implements IEventsCacheAsync { @@ -33,7 +32,7 @@ export default class EventsCacheInRedis implements IEventsCacheAsync { // We use boolean values to signal successful queueing .then(() => true) .catch(err => { - this.log.error(logPrefix + `Error adding event to queue: ${err}.`); + this.log.error(LOG_PREFIX + `Error adding event to queue: ${err}.`); return false; }); } diff --git a/src/storages/inRedis/RedisAdapter.ts b/src/storages/inRedis/RedisAdapter.ts index 7d0636bc..de0abee3 100644 --- a/src/storages/inRedis/RedisAdapter.ts +++ b/src/storages/inRedis/RedisAdapter.ts @@ -5,7 +5,7 @@ import { _Set, setToArray, ISet } from '../../utils/lang/sets'; import thenable from '../../utils/promise/thenable'; import timeout from '../../utils/promise/timeout'; -const logPrefix = 'storage:redis-adapter: '; +const LOG_PREFIX = 'storage:redis-adapter: '; // If we ever decide to fully wrap every method, there's a Commander.getBuiltinCommands from ioredis. const METHODS_TO_PROMISE_WRAP = ['set', 'exec', 'del', 'get', 'keys', 'sadd', 'srem', 'sismember', 'smembers', 'incr', 'rpush', 'pipeline', 'expire', 'mget']; @@ -55,16 +55,16 @@ export default class RedisAdapter extends ioredis { _listenToEvents() { this.once('ready', () => { const commandsCount = this._notReadyCommandsQueue ? this._notReadyCommandsQueue.length : 0; - this.log.info(logPrefix + `Redis connection established. Queued commands: ${commandsCount}.`); + this.log.info(LOG_PREFIX + `Redis connection established. Queued commands: ${commandsCount}.`); this._notReadyCommandsQueue && this._notReadyCommandsQueue.forEach(queued => { - this.log.info(logPrefix + `Executing queued ${queued.name} command.`); + this.log.info(LOG_PREFIX + `Executing queued ${queued.name} command.`); queued.command().then(queued.resolve).catch(queued.reject); }); // After the SDK is ready for the first time we'll stop queueing commands. This is just so we can keep handling BUR for them. this._notReadyCommandsQueue = undefined; }); this.once('close', () => { - this.log.info(logPrefix + 'Redis connection closed.'); + this.log.info(LOG_PREFIX + 'Redis connection closed.'); }); } @@ -78,7 +78,7 @@ export default class RedisAdapter extends ioredis { const params = arguments; function commandWrapper() { - instance.log.debug(logPrefix + `Executing ${method}.`); + instance.log.debug(LOG_PREFIX + `Executing ${method}.`); // Return original method const result = originalMethod.apply(instance, params); @@ -93,7 +93,7 @@ export default class RedisAdapter extends ioredis { result.then(cleanUpRunningCommandsCb, cleanUpRunningCommandsCb); return timeout(instance._options.operationTimeout, result).catch(err => { - instance.log.error(logPrefix + `${method} operation threw an error or exceeded configured timeout of ${instance._options.operationTimeout}ms. Message: ${err}`); + instance.log.error(LOG_PREFIX + `${method} operation threw an error or exceeded configured timeout of ${instance._options.operationTimeout}ms. Message: ${err}`); // Handling is not the adapter responsibility. throw err; }); @@ -126,19 +126,19 @@ export default class RedisAdapter extends ioredis { setTimeout(function deferedDisconnect() { if (instance._runningCommands.size > 0) { - instance.log.info(logPrefix + `Attempting to disconnect but there are ${instance._runningCommands.size} commands still waiting for resolution. Defering disconnection until those finish.`); + instance.log.info(LOG_PREFIX + `Attempting to disconnect but there are ${instance._runningCommands.size} commands still waiting for resolution. Defering disconnection until those finish.`); Promise.all(setToArray(instance._runningCommands)) .then(() => { - instance.log.debug(logPrefix + 'Pending commands finished successfully, disconnecting.'); + instance.log.debug(LOG_PREFIX + 'Pending commands finished successfully, disconnecting.'); originalMethod.apply(instance, params); }) .catch(e => { - instance.log.warn(logPrefix + `Pending commands finished with error: ${e}. Proceeding with disconnection.`); + instance.log.warn(LOG_PREFIX + `Pending commands finished with error: ${e}. Proceeding with disconnection.`); originalMethod.apply(instance, params); }); } else { - instance.log.debug(logPrefix + 'No commands pending execution, disconnect.'); + instance.log.debug(LOG_PREFIX + 'No commands pending execution, disconnect.'); // Nothing pending, just proceed. originalMethod.apply(instance, params); } diff --git a/src/storages/inRedis/SplitsCacheInRedis.ts b/src/storages/inRedis/SplitsCacheInRedis.ts index c601b549..7d6503e2 100644 --- a/src/storages/inRedis/SplitsCacheInRedis.ts +++ b/src/storages/inRedis/SplitsCacheInRedis.ts @@ -3,8 +3,7 @@ import KeyBuilderSS from '../KeyBuilderSS'; import { ISplitsCacheAsync } from '../types'; import { Redis } from 'ioredis'; import { ILogger } from '../../logger/types'; - -const logPrefix = 'storage:redis: '; +import { LOG_PREFIX } from './constants'; /** * Discard errors for an answer of multiple operations. @@ -83,7 +82,7 @@ export default class SplitsCacheInRedis implements ISplitsCacheAsync { */ getSplit(name: string): Promise { if (this.redisError) { - this.log.error(logPrefix + this.redisError); + this.log.error(LOG_PREFIX + this.redisError); throw this.redisError; } @@ -137,14 +136,14 @@ export default class SplitsCacheInRedis implements ISplitsCacheAsync { .then((ttCount: string | null | number) => { ttCount = parseInt(ttCount as string, 10); if (!isFiniteNumber(ttCount) || ttCount < 0) { - this.log.info(logPrefix + `Could not validate traffic type existance of ${trafficType} due to data corruption of some sorts.`); + this.log.info(LOG_PREFIX + `Could not validate traffic type existance of ${trafficType} due to data corruption of some sorts.`); return false; } return ttCount > 0; }) .catch(e => { - this.log.error(logPrefix + `Could not validate traffic type existance of ${trafficType} due to an error: ${e}.`); + this.log.error(LOG_PREFIX + `Could not validate traffic type existance of ${trafficType} due to an error: ${e}.`); // If there is an error, bypass the validation so the event can get tracked. return true; }); @@ -169,7 +168,7 @@ export default class SplitsCacheInRedis implements ISplitsCacheAsync { */ getSplits(names: string[]): Promise> { if (this.redisError) { - this.log.error(logPrefix + this.redisError); + this.log.error(LOG_PREFIX + this.redisError); throw this.redisError; } @@ -183,7 +182,7 @@ export default class SplitsCacheInRedis implements ISplitsCacheAsync { return Promise.resolve(splits); }) .catch(e => { - this.log.error(logPrefix + `Could not grab splits due to an error: ${e}.`); + this.log.error(LOG_PREFIX + `Could not grab splits due to an error: ${e}.`); return Promise.reject(e); }); } diff --git a/src/storages/inRedis/__tests__/RedisAdapter.spec.ts b/src/storages/inRedis/__tests__/RedisAdapter.spec.ts index ba369d8c..9557162d 100644 --- a/src/storages/inRedis/__tests__/RedisAdapter.spec.ts +++ b/src/storages/inRedis/__tests__/RedisAdapter.spec.ts @@ -6,7 +6,7 @@ import { _Set, setToArray } from '../../../utils/lang/sets'; // Mocking sdkLogger import { loggerMock } from '../../../logger/__tests__/sdkLogger.mock'; -const logPrefix = 'storage:redis-adapter: '; +const LOG_PREFIX = 'storage:redis-adapter: '; // Mocking ioredis @@ -188,7 +188,7 @@ describe('STORAGE Redis Adapter', () => { secondCallArgs[1](); // Execute the callback for "close" expect(loggerMock.info).toBeCalledTimes(1); // The callback for the "close" event will only log info to the user about what is going on. - expect(loggerMock.info.mock.calls[0]).toEqual([logPrefix + 'Redis connection closed.']); // The callback for the "close" event will only log info to the user about what is going on. + expect(loggerMock.info.mock.calls[0]).toEqual([LOG_PREFIX + 'Redis connection closed.']); // The callback for the "close" event will only log info to the user about what is going on. loggerMock.info.mockClear(); expect(loggerMock.info).not.toBeCalled(); // Control assertion @@ -198,7 +198,7 @@ describe('STORAGE Redis Adapter', () => { firstCallArgs[1](); expect(loggerMock.info).toBeCalledTimes(1); // The callback for the "ready" event will inform the user about the trigger. - expect(loggerMock.info).toBeCalledWith(logPrefix + 'Redis connection established. Queued commands: 0.'); // The callback for the "ready" event will inform the user about the trigger. + expect(loggerMock.info).toBeCalledWith(LOG_PREFIX + 'Redis connection established. Queued commands: 0.'); // The callback for the "ready" event will inform the user about the trigger. expect(instance._notReadyCommandsQueue).toBe(undefined); // After the DB is ready, it will clean up the offline commands queue so we do not queue commands anymore. // Don't do this at home @@ -222,9 +222,9 @@ describe('STORAGE Redis Adapter', () => { expect(loggerMock.info).toBeCalledTimes(3); // If we had queued commands, it will log the event (1 call) as well as each executed command (n calls). expect(loggerMock.info.mock.calls).toEqual([ - [logPrefix + 'Redis connection established. Queued commands: 2.'], // The callback for the "ready" event will inform the user about the trigger and the amount of queued commands. - [logPrefix + 'Executing queued GET command.'], // If we had queued, it will log the event as well as each executed command. - [logPrefix + 'Executing queued SET command.'] // If we had queued commands, it will log the event as well as each executed command. + [LOG_PREFIX + 'Redis connection established. Queued commands: 2.'], // The callback for the "ready" event will inform the user about the trigger and the amount of queued commands. + [LOG_PREFIX + 'Executing queued GET command.'], // If we had queued, it will log the event as well as each executed command. + [LOG_PREFIX + 'Executing queued SET command.'] // If we had queued commands, it will log the event as well as each executed command. ]); expect(queuedGetCommand.command).toBeCalledTimes(1); // It will execute each queued command. @@ -287,7 +287,7 @@ describe('STORAGE Redis Adapter', () => { commandTimeoutResolver.rej('test'); setTimeout(() => { // Allow the promises to tick. expect(instance._runningCommands.has(commandTimeoutResolver.originalPromise)).toBe(false); // After a command finishes with error, it's promise is removed from the instance._runningCommands queue. - expect(loggerMock.error.mock.calls[index]).toEqual([`${logPrefix}${methodName} operation threw an error or exceeded configured timeout of 5000ms. Message: test`]); // The log error method should be called with the corresponding messages, depending on the method, error and operationTimeout. + expect(loggerMock.error.mock.calls[index]).toEqual([`${LOG_PREFIX}${methodName} operation threw an error or exceeded configured timeout of 5000ms. Message: test`]); // The log error method should be called with the corresponding messages, depending on the method, error and operationTimeout. }, 0); }); @@ -334,7 +334,7 @@ describe('STORAGE Redis Adapter', () => { expect(ioredisMock.disconnect).not.toBeCalled(); // Original method should not be called right away. setTimeout(() => { // o queued commands timeout wrapper. - expect(loggerMock.debug.mock.calls).toEqual([[logPrefix + 'No commands pending execution, disconnect.']]); + expect(loggerMock.debug.mock.calls).toEqual([[LOG_PREFIX + 'No commands pending execution, disconnect.']]); expect(ioredisMock.disconnect).toBeCalledTimes(1); // Original method should have been called once, asynchronously loggerMock.debug.mockClear(); ioredisMock.disconnect.mockClear(); @@ -347,11 +347,11 @@ describe('STORAGE Redis Adapter', () => { rejectedPromise.catch(() => { }); // Swallow the unhandled to avoid unhandledRejection warns setTimeout(() => { // queued with rejection timeout wrapper - expect(loggerMock.info.mock.calls).toEqual([[logPrefix + 'Attempting to disconnect but there are 2 commands still waiting for resolution. Defering disconnection until those finish.']]); + expect(loggerMock.info.mock.calls).toEqual([[LOG_PREFIX + 'Attempting to disconnect but there are 2 commands still waiting for resolution. Defering disconnection until those finish.']]); Promise.all(setToArray(instance._runningCommands)).catch(e => { setImmediate(() => { // Allow the callback to execute before checking. - expect(loggerMock.warn.mock.calls[0]).toEqual([`${logPrefix}Pending commands finished with error: ${e}. Proceeding with disconnection.`]); // Should warn about the error but tell user that will disconnect anyways. + expect(loggerMock.warn.mock.calls[0]).toEqual([`${LOG_PREFIX}Pending commands finished with error: ${e}. Proceeding with disconnection.`]); // Should warn about the error but tell user that will disconnect anyways. expect(ioredisMock.disconnect).toBeCalledTimes(1); // Original method should have been called once, asynchronously loggerMock.info.mockClear(); @@ -367,11 +367,11 @@ describe('STORAGE Redis Adapter', () => { instance._runningCommands.add(Promise.resolve()); setTimeout(() => { - expect(loggerMock.info.mock.calls).toEqual([[logPrefix + 'Attempting to disconnect but there are 4 commands still waiting for resolution. Defering disconnection until those finish.']]); + expect(loggerMock.info.mock.calls).toEqual([[LOG_PREFIX + 'Attempting to disconnect but there are 4 commands still waiting for resolution. Defering disconnection until those finish.']]); Promise.all(setToArray(instance._runningCommands)).then(() => { // This one will go through success path setImmediate(() => { - expect(loggerMock.debug.mock.calls).toEqual([[logPrefix + 'Pending commands finished successfully, disconnecting.']]); + expect(loggerMock.debug.mock.calls).toEqual([[LOG_PREFIX + 'Pending commands finished successfully, disconnecting.']]); expect(ioredisMock.disconnect).toBeCalledTimes(1); // Original method should have been called once, asynchronously }); }); diff --git a/src/storages/inRedis/constants.ts b/src/storages/inRedis/constants.ts new file mode 100644 index 00000000..a3856844 --- /dev/null +++ b/src/storages/inRedis/constants.ts @@ -0,0 +1 @@ +export const LOG_PREFIX = 'storage:redis: '; diff --git a/src/storages/pluggable/EventsCachePluggable.ts b/src/storages/pluggable/EventsCachePluggable.ts index eef10dd1..2af117d7 100644 --- a/src/storages/pluggable/EventsCachePluggable.ts +++ b/src/storages/pluggable/EventsCachePluggable.ts @@ -3,7 +3,7 @@ import { IRedisMetadata } from '../../dtos/types'; import KeyBuilderSS from '../KeyBuilderSS'; import { SplitIO } from '../../types'; import { ILogger } from '../../logger/types'; -import { logPrefix } from './constants'; +import { LOG_PREFIX } from './constants'; export class EventsCachePluggable implements IEventsCacheAsync { @@ -29,10 +29,13 @@ export class EventsCachePluggable implements IEventsCacheAsync { return this.wrapper.pushItems( this.keys.buildEventsKey(), [this._toJSON(eventData)] - ).catch(e => { - this.log.error(logPrefix + ` Error adding event to queue: ${e}.`); - return false; - }); + ) + // We use boolean values to signal successful queueing + .then(() => true) + .catch(e => { + this.log.error(LOG_PREFIX + ` Error adding event to queue: ${e}.`); + return false; + }); } private _toJSON(eventData: SplitIO.EventData): string { diff --git a/src/storages/pluggable/ImpressionsCachePluggable.ts b/src/storages/pluggable/ImpressionsCachePluggable.ts index 643b64de..31436c43 100644 --- a/src/storages/pluggable/ImpressionsCachePluggable.ts +++ b/src/storages/pluggable/ImpressionsCachePluggable.ts @@ -3,7 +3,7 @@ import { IRedisMetadata } from '../../dtos/types'; import { ImpressionDTO } from '../../types'; import KeyBuilderSS from '../KeyBuilderSS'; import { ILogger } from '../../logger/types'; -import { logPrefix } from './constants'; +import { LOG_PREFIX } from './constants'; export class ImpressionsCachePluggable implements IImpressionsCacheAsync { @@ -29,10 +29,13 @@ export class ImpressionsCachePluggable implements IImpressionsCacheAsync { return this.wrapper.pushItems( this.keys.buildImpressionsKey(), this._toJSON(impressions) - ).catch((e) => { - this.log.error(logPrefix + ` Error adding event to queue: ${e}.`); - return false; - }); + ) + // We use boolean values to signal successful queueing + .then(() => true) + .catch((e) => { + this.log.error(LOG_PREFIX + ` Error adding event to queue: ${e}.`); + return false; + }); } private _toJSON(impressions: ImpressionDTO[]): string[] { diff --git a/src/storages/pluggable/SegmentsCachePluggable.ts b/src/storages/pluggable/SegmentsCachePluggable.ts index 67fc39be..612b5ff0 100644 --- a/src/storages/pluggable/SegmentsCachePluggable.ts +++ b/src/storages/pluggable/SegmentsCachePluggable.ts @@ -4,7 +4,7 @@ import { isNaNNumber } from '../../utils/lang'; import KeyBuilder from '../KeyBuilder'; import { ICustomStorageWrapper, ISegmentsCacheAsync } from '../types'; import { ILogger } from '../../logger/types'; -import { logPrefix } from './constants'; +import { LOG_PREFIX } from './constants'; /** * ISegmentsCacheAsync implementation for pluggable storages. @@ -49,7 +49,7 @@ export class SegmentsCachePluggable implements ISegmentsCacheAsync { const i = parseInt(value as string, 10); return isNaNNumber(i) ? -1 : i; }).catch((e) => { - this.log.error(logPrefix + 'Could not retrieve changeNumber from segments storage. Error: ' + e); + this.log.error(LOG_PREFIX + 'Could not retrieve changeNumber from segments storage. Error: ' + e); return -1; }); } diff --git a/src/storages/pluggable/__tests__/EventsCachePluggable.spec.ts b/src/storages/pluggable/__tests__/EventsCachePluggable.spec.ts index e7f168fb..9bb8cef0 100644 --- a/src/storages/pluggable/__tests__/EventsCachePluggable.spec.ts +++ b/src/storages/pluggable/__tests__/EventsCachePluggable.spec.ts @@ -30,15 +30,15 @@ describe('PLUGGABLE EVENTS CACHE', () => { // @ts-expect-error expect(await cache.track(fakeEvent3)).toBe(true); // If the queueing operation was successful, it should resolve the returned promise with "true" - const redisValues = wrapperMock._queues[key]; + const values = wrapperMock._cache[key]; - expect(redisValues.length).toBe(3); // After pushing we should have on Redis as many events as we have stored. - expect(typeof redisValues[0]).toBe('string'); // All elements should be strings since those are stringified JSONs. - expect(typeof redisValues[1]).toBe('string'); // All elements should be strings since those are stringified JSONs. - expect(typeof redisValues[2]).toBe('string'); // All elements should be strings since those are stringified JSONs. + expect(values.length).toBe(3); // After pushing we should have as many events as we have stored. + expect(typeof values[0]).toBe('string'); // All elements should be strings since those are stringified JSONs. + expect(typeof values[1]).toBe('string'); // All elements should be strings since those are stringified JSONs. + expect(typeof values[2]).toBe('string'); // All elements should be strings since those are stringified JSONs. const findMatchingElem = (event: any) => { - return find(redisValues, elem => { + return find(values, elem => { const parsedElem = JSON.parse(elem); return isEqual(parsedElem.e, event) && isEqual(parsedElem.m, fakeRedisMetadata); }); @@ -48,9 +48,9 @@ describe('PLUGGABLE EVENTS CACHE', () => { const foundEv1 = findMatchingElem(fakeEvent1); const foundEv2 = findMatchingElem(fakeEvent2); const foundEv3 = findMatchingElem(fakeEvent3); - expect(foundEv1).not.toBe(undefined); // Events stored on redis matched the values we are expecting. - expect(foundEv2).not.toBe(undefined); // Events stored on redis matched the values we are expecting. - expect(foundEv3).not.toBe(undefined); // Events stored on redis matched the values we are expecting. + expect(foundEv1).not.toBe(undefined); // stored events matched the values we are expecting. + expect(foundEv2).not.toBe(undefined); // stored events matched the values we are expecting. + expect(foundEv3).not.toBe(undefined); // stored events matched the values we are expecting. wrapperMock.mockClear(); }); @@ -58,7 +58,7 @@ describe('PLUGGABLE EVENTS CACHE', () => { test('`track` method result should not reject if wrapper operation fails', async () => { const wrapperMock = wrapperMockFactory(); // @ts-ignore. I'll use a "bad" queue to force an exception with the pushItems wrapper operation. - wrapperMock._queues['non-list-key'] = 10; + wrapperMock._cache['non-list-key'] = 10; // @ts-expect-error. wrapperMock is adapted this time to properly handle unexpected exceptions const faultyCache = new EventsCachePluggable(loggerMock, { buildEventsKey: () => 'non-list-key' diff --git a/src/storages/pluggable/__tests__/ImpressionsCachePluggable.spec.ts b/src/storages/pluggable/__tests__/ImpressionsCachePluggable.spec.ts index 42dbeedb..00a88d95 100644 --- a/src/storages/pluggable/__tests__/ImpressionsCachePluggable.spec.ts +++ b/src/storages/pluggable/__tests__/ImpressionsCachePluggable.spec.ts @@ -50,12 +50,12 @@ describe('PLUGGABLE IMPRESSIONS CACHE', () => { expect(await cache.track([o1])).toBe(true); // result should resolve to true - let state = wrapperMock._queues[impressionsKey]; + let state = wrapperMock._cache[impressionsKey]; expect(state.length).toBe(1); // impression should be stored expect(await cache.track([o2, o3])).toBe(true); - state = wrapperMock._queues[impressionsKey]; + state = wrapperMock._cache[impressionsKey]; expect(state.length).toBe(3); // This is testing both the track and the toJSON method. expect(state[0]).toBe(JSON.stringify({ @@ -75,7 +75,7 @@ describe('PLUGGABLE IMPRESSIONS CACHE', () => { test('`track` method result should not reject if wrapper operation fails', async () => { // make wrapper operation fail - wrapperMock.pushItems.mockImplementation(()=>Promise.reject()); + wrapperMock.pushItems.mockImplementation(() => Promise.reject()); const cache = new ImpressionsCachePluggable(loggerMock, keys, wrapperMock, testMeta); expect(await cache.track([o1])).toBe(false); // result should resolve to false diff --git a/src/storages/pluggable/__tests__/wrapper.mock.ts b/src/storages/pluggable/__tests__/wrapper.mock.ts index 352c67cb..8aaf3a40 100644 --- a/src/storages/pluggable/__tests__/wrapper.mock.ts +++ b/src/storages/pluggable/__tests__/wrapper.mock.ts @@ -3,12 +3,11 @@ import { startsWith, toNumber } from '../../../utils/lang'; // Creates an in memory ICustomStorageWrapper implementation with Jest mocks export function wrapperMockFactory() { + /** Holds items and list of items */ let _cache: Record = {}; - let _queues: Record = {}; return { _cache, - _queues, get: jest.fn((key => { return Promise.resolve(key in _cache ? _cache[key] : null); @@ -57,19 +56,26 @@ export function wrapperMockFactory() { getMany: jest.fn((keys: string[]) => { return Promise.resolve(keys.map(key => _cache[key] ? _cache[key] : null)); }), - pushItems: jest.fn((key: string, items: string[]) => { - if (!(key in _queues)) _queues[key] = []; - _queues[key].push(...items); - return Promise.resolve(true); + pushItems: jest.fn((key: string, items: string[]) => { // @ts-ignore + if (!(key in _cache)) _cache[key] = []; + const list = _cache[key]; + if (Array.isArray(list)) { + list.push(...items); + return Promise.resolve(); + } + return Promise.reject('key is not a list'); }), popItems: jest.fn((key: string, count: number) => { - return Promise.resolve(key in _queues ? _queues[key].splice(0, count) : []); + const list = _cache[key]; + return Promise.resolve(Array.isArray(list) ? list.splice(0, count) : []); }), getItemsCount: jest.fn((key: string) => { - return Promise.resolve(key in _queues ? _queues[key].length : 0); + const list = _cache[key]; + return Promise.resolve(Array.isArray(list) ? list.length : 0); }), itemContains: jest.fn((key: string, item: string) => { - return Promise.resolve(key in _queues && _queues[key].indexOf(item) > -1 ? true : false); + const list = _cache[key]; + return Promise.resolve(Array.isArray(list) && list.indexOf(item) > -1 ? true : false); }), // always connects and close @@ -78,7 +84,6 @@ export function wrapperMockFactory() { mockClear() { this._cache = {}; - this._queues = {}; this.get.mockClear(); this.set.mockClear(); this.del.mockClear(); diff --git a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts index c6292f87..430c3663 100644 --- a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts +++ b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts @@ -49,7 +49,6 @@ export const wrapperWithValuesToSanitize = { connect: () => Promise.resolve(1), getAndSet: () => Promise.resolve(true), getByPrefix: () => Promise.resolve(['1', null, false, true, '2', null]), - pushItems: () => Promise.resolve('false'), popItems: () => Promise.resolve('invalid array'), getItemsCount: () => Promise.resolve('10'), itemContains: () => Promise.resolve('true'), @@ -66,7 +65,6 @@ const SANITIZED_RESULTS = { connect: false, getAndSet: 'true', getByPrefix: ['1', '2'], - pushItems: false, popItems: [], getItemsCount: 10, itemContains: true, @@ -84,7 +82,7 @@ const VALID_METHOD_CALLS = { 'close': [], 'getAndSet': ['some_key', 'some_value'], 'getByPrefix': ['some_prefix'], - 'pushItems': ['some_key', ['item1', 'item2']], + 'pushItems': ['some_key_list', ['item1', 'item2']], 'popItems': ['some_key'], 'getItemsCount': ['some_key'], 'itemContains': ['some_key', 'some_value'], diff --git a/src/storages/pluggable/constants.ts b/src/storages/pluggable/constants.ts index dbac131e..1f845f45 100644 --- a/src/storages/pluggable/constants.ts +++ b/src/storages/pluggable/constants.ts @@ -1 +1 @@ -export const logPrefix = 'storage:pluggable:'; +export const LOG_PREFIX = 'storage:pluggable:'; diff --git a/src/storages/pluggable/wrapperAdapter.ts b/src/storages/pluggable/wrapperAdapter.ts index 37fa666d..63eb5a4b 100644 --- a/src/storages/pluggable/wrapperAdapter.ts +++ b/src/storages/pluggable/wrapperAdapter.ts @@ -3,7 +3,7 @@ import { sanitizeBoolean as sBoolean } from '../../evaluator/value/sanitize'; import { ILogger } from '../../logger/types'; import { SplitError } from '../../utils/lang/errors'; import { ICustomStorageWrapper } from '../types'; -import { logPrefix } from './constants'; +import { LOG_PREFIX } from './constants'; // Sanitizers return the given value if it is of the expected type, or a new sanitized one if invalid. @@ -50,7 +50,7 @@ const METHODS_TO_PROMISE_WRAP: [string, undefined | { (val: any): any, type: str ['close', undefined], ['getAndSet', sanitizeNullableString], ['getByPrefix', sanitizeArray], - ['pushItems', sanitizeBoolean], + ['pushItems', undefined], ['popItems', sanitizeArray], ['getItemsCount', sanitizeNumber], ['itemContains', sanitizeBoolean], @@ -70,9 +70,9 @@ export function wrapperAdapter(log: ILogger, wrapper: ICustomStorageWrapper): IC METHODS_TO_PROMISE_WRAP.forEach(([method, sanitizer]) => { - // Logs error and wraps it into an SplitError object + // Logs error and wraps it into an SplitError object (required to handle user callback errors in SDK readiness events) function handleError(e: any) { - log.error(`${logPrefix} Wrapper '${method}' operation threw an error. Message: ${e}`); + log.error(`${LOG_PREFIX} Wrapper '${method}' operation threw an error. Message: ${e}`); return Promise.reject(new SplitError(e)); } @@ -84,7 +84,7 @@ export function wrapperAdapter(log: ILogger, wrapper: ICustomStorageWrapper): IC const sanitizedValue = sanitizer(value); // if value had to be sanitized, log a warning - if (sanitizedValue !== value) log.warn(`${logPrefix} Attempted to sanitize return value [${value}] of wrapper '${method}' operation which should be of type [${sanitizer.type}]. Sanitized and processed value => [${sanitizedValue}]`); + if (sanitizedValue !== value) log.warn(`${LOG_PREFIX} Attempted to sanitize return value [${value}] of wrapper '${method}' operation which should be of type [${sanitizer.type}]. Sanitized and processed value => [${sanitizedValue}]`); return sanitizedValue; })).catch(handleError); diff --git a/src/storages/types.ts b/src/storages/types.ts index f7f23eda..96ea2705 100644 --- a/src/storages/types.ts +++ b/src/storages/types.ts @@ -61,20 +61,24 @@ export interface ICustomStorageWrapper { */ getMany: (keys: string[]) => Promise<(string | null)[]> /** - * Push given `items` to `key` queue. + * Push given `items` to `key` list. If key does not exist, an empty list is created for the key before pushing the items. + * Return a promise that resolves if the operation success, + * or rejects if the operation fails, for example, if there is a connectin error or the key holds a value that is not a list. */ - pushItems: (key: string, items: string[]) => Promise + pushItems: (key: string, items: string[]) => Promise /** * Pop `count` number of items from `key` queue. - * Return a promise that resolves with the list of removed items the removed members, or an empty array when key does not exist. + * Return a promise that resolves with the list of removed items the removed members, + * or an empty array when key does not exist or is not a list. */ popItems: (key: string, count: number) => Promise /** - * Return a promise that resolves with the number of items at the `key` queue, or 0 when key does not exist. + * Return a promise that resolves with the number of items at the `key` queue, or 0 when key does not exist or is not a list. */ getItemsCount: (key: string) => Promise /** - * Return a promise that resolves with true boolean value if `item` is a member of the set stored at `key`, or false otherwise. + * Return a promise that resolves with true boolean value if `item` is a member of the list stored at `key`, + * or false if it is not a member or key does not exist or is not a list. */ itemContains: (key: string, item: string) => Promise From 05ccd7df959dd45ce6288cc7099a2f24d33d2fe6 Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Thu, 29 Apr 2021 21:08:32 -0300 Subject: [PATCH 11/14] some fixes --- .../pluggable/SplitsCachePluggable.ts | 8 ++--- .../pluggable/__tests__/wrapper.mock.ts | 10 +++---- src/storages/pluggable/wrapperAdapter.ts | 30 ++++++++++--------- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/storages/pluggable/SplitsCachePluggable.ts b/src/storages/pluggable/SplitsCachePluggable.ts index cc035cea..9362aaab 100644 --- a/src/storages/pluggable/SplitsCachePluggable.ts +++ b/src/storages/pluggable/SplitsCachePluggable.ts @@ -4,7 +4,7 @@ import { ICustomStorageWrapper, ISplitsCacheAsync } from '../types'; import { ILogger } from '../../logger/types'; import { usesSegments } from '../AbstractSplitsCacheSync'; import { ISplit } from '../../dtos/types'; -import { logPrefix } from './constants'; +import { LOG_PREFIX } from './constants'; import { SplitError } from '../../utils/lang/errors'; /** @@ -180,13 +180,13 @@ export class SplitsCachePluggable implements ISplitsCacheAsync { ttCount = parseInt(ttCount as string, 10); if (!isFiniteNumber(ttCount) || ttCount < 0) { - this.log.info(logPrefix + `Could not validate traffic type existence of ${trafficType} due to data corruption of some sorts.`); + this.log.info(LOG_PREFIX + `Could not validate traffic type existence of ${trafficType} due to data corruption of some sorts.`); return false; } return ttCount > 0; }).catch(e => { - this.log.error(logPrefix + `Could not validate traffic type existence of ${trafficType} due to an error: ${e}.`); + this.log.error(LOG_PREFIX + `Could not validate traffic type existence of ${trafficType} due to an error: ${e}.`); // If there is an error, bypass the validation so the event can get tracked. return true; }); @@ -212,7 +212,7 @@ export class SplitsCachePluggable implements ISplitsCacheAsync { return isNaNNumber(i) ? -1 : i; }).catch((e) => { - this.log.error(logPrefix + 'Could not retrieve changeNumber from storage. Error: ' + e); + this.log.error(LOG_PREFIX + 'Could not retrieve changeNumber from storage. Error: ' + e); return -1; }); } diff --git a/src/storages/pluggable/__tests__/wrapper.mock.ts b/src/storages/pluggable/__tests__/wrapper.mock.ts index 8aaf3a40..60e8b772 100644 --- a/src/storages/pluggable/__tests__/wrapper.mock.ts +++ b/src/storages/pluggable/__tests__/wrapper.mock.ts @@ -4,13 +4,13 @@ import { startsWith, toNumber } from '../../../utils/lang'; export function wrapperMockFactory() { /** Holds items and list of items */ - let _cache: Record = {}; + let _cache: Record = {}; return { _cache, get: jest.fn((key => { - return Promise.resolve(key in _cache ? _cache[key] : null); + return Promise.resolve(key in _cache ? _cache[key] as string : null); })), set: jest.fn((key: string, value: string) => { const result = key in _cache; @@ -18,7 +18,7 @@ export function wrapperMockFactory() { return Promise.resolve(result); }), getAndSet: jest.fn((key: string, value: string) => { - const result = key in _cache ? _cache[key] : null; + const result = key in _cache ? _cache[key] as string : null; _cache[key] = value; return Promise.resolve(result); }), @@ -31,7 +31,7 @@ export function wrapperMockFactory() { return Promise.resolve(Object.keys(_cache).filter(key => startsWith(key, prefix))); }), getByPrefix: jest.fn((prefix: string) => { - return Promise.resolve(Object.keys(_cache).filter(key => startsWith(key, prefix)).map(key => _cache[key])); + return Promise.resolve(Object.keys(_cache).filter(key => startsWith(key, prefix)).map(key => _cache[key] as string)); }), incr: jest.fn((key: string) => { if (key in _cache) { @@ -54,7 +54,7 @@ export function wrapperMockFactory() { return Promise.resolve(true); }), getMany: jest.fn((keys: string[]) => { - return Promise.resolve(keys.map(key => _cache[key] ? _cache[key] : null)); + return Promise.resolve(keys.map(key => _cache[key] ? _cache[key] as string : null)); }), pushItems: jest.fn((key: string, items: string[]) => { // @ts-ignore if (!(key in _cache)) _cache[key] = []; diff --git a/src/storages/pluggable/wrapperAdapter.ts b/src/storages/pluggable/wrapperAdapter.ts index 63eb5a4b..a7049d9e 100644 --- a/src/storages/pluggable/wrapperAdapter.ts +++ b/src/storages/pluggable/wrapperAdapter.ts @@ -6,6 +6,7 @@ import { ICustomStorageWrapper } from '../types'; import { LOG_PREFIX } from './constants'; // Sanitizers return the given value if it is of the expected type, or a new sanitized one if invalid. +// @TODO review or remove sanitizers. might be expensive for producer methods function sanitizeBoolean(val: any): boolean { return sBoolean(val) || false; @@ -17,41 +18,42 @@ function sanitizeNumber(val: any): number { } sanitizeNumber.type = 'number'; -function sanitizeArray(val: any): string[] { +function sanitizeArrayOfStrings(val: any): string[] { if (!Array.isArray(val)) return []; // if not an array, returns a new empty one if (val.every(isString)) return val; // if all items are valid, return the given array - return val.filter(isString); // otherwise, return a new array filtering the invalid items + return val.map(toString); // otherwise, return a new array with items sanitized } -sanitizeArray.type = 'Array'; +sanitizeArrayOfStrings.type = 'Array'; function sanitizeNullableString(val: any): string | null { if (val === null) return val; - return toString(val); + return toString(val); // if not null, sanitize value as an string } sanitizeNullableString.type = 'string | null'; -function sanitizeArrayOfNullableString(val: any): (string | null)[] { - const isStringOrNull = (v: any) => v === null || isString(v); +function sanitizeArrayOfNullableStrings(val: any): (string | null)[] { if (!Array.isArray(val)) return []; - if (val.every(isStringOrNull)) return val; - return val.filter(isStringOrNull); + if (val.every(v => v === null || isString(v))) return val; // if all items are string or null, return the given array + return val.map(v => v !== null ? toString(v) : null); // otherwise, return a new array with items sanitized } -sanitizeArrayOfNullableString.type = 'Array'; +sanitizeArrayOfNullableStrings.type = 'Array'; const METHODS_TO_PROMISE_WRAP: [string, undefined | { (val: any): any, type: string }][] = [ ['get', sanitizeNullableString], ['set', sanitizeBoolean], + ['getAndSet', sanitizeNullableString], ['del', sanitizeBoolean], - ['getKeysByPrefix', sanitizeArray], + ['getKeysByPrefix', sanitizeArrayOfStrings], + ['getByPrefix', sanitizeArrayOfStrings], ['incr', sanitizeBoolean], ['decr', sanitizeBoolean], - ['getMany', sanitizeArrayOfNullableString], + ['getMany', sanitizeArrayOfNullableStrings], ['connect', sanitizeBoolean], ['close', undefined], - ['getAndSet', sanitizeNullableString], - ['getByPrefix', sanitizeArray], + + ['pushItems', undefined], - ['popItems', sanitizeArray], + ['popItems', sanitizeArrayOfStrings], ['getItemsCount', sanitizeNumber], ['itemContains', sanitizeBoolean], ]; From 226155369f4554eba9b3022621a8a2d4652a3318 Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Thu, 29 Apr 2021 21:56:41 -0300 Subject: [PATCH 12/14] test fixes --- .../__tests__/EventsCachePluggable.spec.ts | 2 +- .../__tests__/SplitsCachePluggable.spec.ts | 17 ++++++----------- .../pluggable/__tests__/wrapperAdapter.spec.ts | 6 +++--- src/storages/pluggable/wrapperAdapter.ts | 6 +++--- src/sync/polling/syncTasks/splitsSyncTask.ts | 2 +- 5 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/storages/pluggable/__tests__/EventsCachePluggable.spec.ts b/src/storages/pluggable/__tests__/EventsCachePluggable.spec.ts index 9bb8cef0..dfac96f6 100644 --- a/src/storages/pluggable/__tests__/EventsCachePluggable.spec.ts +++ b/src/storages/pluggable/__tests__/EventsCachePluggable.spec.ts @@ -3,7 +3,7 @@ import isEqual from 'lodash/isEqual'; import { loggerMock } from '../../../logger/__tests__/sdkLogger.mock'; import KeyBuilderSS from '../../KeyBuilderSS'; import { EventsCachePluggable } from '../EventsCachePluggable'; -import { metadataBuilder } from '../../inRedis'; +import { metadataBuilder } from '../../metadataBuilder'; import { wrapperMockFactory } from './wrapper.mock'; import { wrapperAdapter } from '../wrapperAdapter'; diff --git a/src/storages/pluggable/__tests__/SplitsCachePluggable.spec.ts b/src/storages/pluggable/__tests__/SplitsCachePluggable.spec.ts index 6e3c1c3e..d7b5ff00 100644 --- a/src/storages/pluggable/__tests__/SplitsCachePluggable.spec.ts +++ b/src/storages/pluggable/__tests__/SplitsCachePluggable.spec.ts @@ -1,7 +1,7 @@ import { SplitsCachePluggable } from '../SplitsCachePluggable'; import KeyBuilder from '../../KeyBuilder'; import { loggerMock } from '../../../logger/__tests__/sdkLogger.mock'; -import { wrapperMock } from './wrapper.mock'; +import { wrapperMockFactory } from './wrapper.mock'; const splitWithUserTT = '{ "trafficTypeName": "user_tt" }'; const splitWithAccountTT = '{ "trafficTypeName": "account_tt" }'; @@ -10,13 +10,8 @@ const keysBuilder = new KeyBuilder(); describe('SPLITS CACHE PLUGGABLE', () => { - afterEach(() => { - loggerMock.mockClear(); - wrapperMock.mockClear(); - }); - test('add/remove/get splits', async () => { - const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMock); + const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMockFactory()); // Assert addSplit and addSplits await cache.addSplits([ @@ -76,7 +71,7 @@ describe('SPLITS CACHE PLUGGABLE', () => { }); test('set/get change number', async () => { - const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMock); + const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMockFactory()); expect(await cache.getChangeNumber()).toBe(-1); // if not set yet, changeNumber is -1 await cache.setChangeNumber(123); @@ -85,7 +80,7 @@ describe('SPLITS CACHE PLUGGABLE', () => { }); test('trafficTypeExists', async () => { - const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMock); + const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMockFactory()); await cache.addSplits([ ['split1', splitWithUserTT], @@ -125,7 +120,7 @@ describe('SPLITS CACHE PLUGGABLE', () => { }); test('usesSegments', async () => { - const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMock); + const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMockFactory()); await cache.addSplits([['split1', splitWithUserTT], ['split2', splitWithAccountTT],]); expect(await cache.usesSegments()).toBe(false); // 0 splits using segments @@ -144,7 +139,7 @@ describe('SPLITS CACHE PLUGGABLE', () => { }); test('killLocally', async () => { - const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMock); + const cache = new SplitsCachePluggable(loggerMock, keysBuilder, wrapperMockFactory()); await cache.addSplit('lol1', splitWithUserTT); await cache.addSplit('lol2', splitWithAccountTT); diff --git a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts index 430c3663..ca8d1dd7 100644 --- a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts +++ b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts @@ -1,7 +1,7 @@ // @ts-nocheck import { loggerMock } from '../../../logger/__tests__/sdkLogger.mock'; import thenable from '../../../utils/promise/thenable'; -import { logPrefix } from '../constants'; +import { LOG_PREFIX } from '../constants'; import { SplitError } from '../../../utils/lang/errors'; /** Mocks */ @@ -61,7 +61,7 @@ const SANITIZED_RESULTS = { getKeysByPrefix: ['1', '2'], incr: false, decr: false, - getMany: ['1', null, '2', null], + getMany: ['1', null, 'false', 'true', '2', null], connect: false, getAndSet: 'true', getByPrefix: ['1', '2'], @@ -129,7 +129,7 @@ describe('Wrapper Adapter', () => { expect(true).toBe(false); // promise shouldn't be resolved } catch (e) { expect(e).toBeInstanceOf(SplitError); - expect(loggerMock.error).toHaveBeenCalledWith(`${logPrefix} Wrapper '${method}' operation threw an error. Message: ${e.message}`); + expect(loggerMock.error).toHaveBeenCalledWith(`${LOG_PREFIX} Wrapper '${method}' operation threw an error. Message: ${e.message}`); } } diff --git a/src/storages/pluggable/wrapperAdapter.ts b/src/storages/pluggable/wrapperAdapter.ts index a7049d9e..ad26d17f 100644 --- a/src/storages/pluggable/wrapperAdapter.ts +++ b/src/storages/pluggable/wrapperAdapter.ts @@ -6,7 +6,7 @@ import { ICustomStorageWrapper } from '../types'; import { LOG_PREFIX } from './constants'; // Sanitizers return the given value if it is of the expected type, or a new sanitized one if invalid. -// @TODO review or remove sanitizers. might be expensive for producer methods +// @TODO review or remove sanitizers. Could be expensive for producer methods function sanitizeBoolean(val: any): boolean { return sBoolean(val) || false; @@ -21,7 +21,7 @@ sanitizeNumber.type = 'number'; function sanitizeArrayOfStrings(val: any): string[] { if (!Array.isArray(val)) return []; // if not an array, returns a new empty one if (val.every(isString)) return val; // if all items are valid, return the given array - return val.map(toString); // otherwise, return a new array with items sanitized + return val.filter(isString); // otherwise, return a new array filtering the invalid items } sanitizeArrayOfStrings.type = 'Array'; @@ -33,7 +33,7 @@ sanitizeNullableString.type = 'string | null'; function sanitizeArrayOfNullableStrings(val: any): (string | null)[] { if (!Array.isArray(val)) return []; - if (val.every(v => v === null || isString(v))) return val; // if all items are string or null, return the given array + if (val.every(v => v === null || isString(v))) return val; return val.map(v => v !== null ? toString(v) : null); // otherwise, return a new array with items sanitized } sanitizeArrayOfNullableStrings.type = 'Array'; diff --git a/src/sync/polling/syncTasks/splitsSyncTask.ts b/src/sync/polling/syncTasks/splitsSyncTask.ts index aec19fe5..557e8c0f 100644 --- a/src/sync/polling/syncTasks/splitsSyncTask.ts +++ b/src/sync/polling/syncTasks/splitsSyncTask.ts @@ -129,7 +129,7 @@ export function splitChangesUpdaterFactory( log.debug(SYNC_SPLITS_SEGMENTS, [mutation.segments.length]); // Write into storage - // @TODO in producer mode, keep consistency of wrapped storate by calling `setChangeNumber` if the other wrapper operations have succeeded + // @TODO call `setChangeNumber` only if the other storage operations have succeeded, in order to keep storage consistency return Promise.all([ // calling first `setChangenumber` method, to perform cache flush if split filter queryString changed splitsCache.setChangeNumber(splitChanges.till), From 5273d10a0356556ac596125fbadab3ff2e619c5c Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Thu, 29 Apr 2021 22:06:57 -0300 Subject: [PATCH 13/14] test fixes --- .../pluggable/__tests__/wrapperAdapter.spec.ts | 2 +- src/storages/pluggable/wrapperAdapter.ts | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts index ca8d1dd7..49ad448d 100644 --- a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts +++ b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts @@ -61,7 +61,7 @@ const SANITIZED_RESULTS = { getKeysByPrefix: ['1', '2'], incr: false, decr: false, - getMany: ['1', null, 'false', 'true', '2', null], + getMany: ['1', null, '2', null], connect: false, getAndSet: 'true', getByPrefix: ['1', '2'], diff --git a/src/storages/pluggable/wrapperAdapter.ts b/src/storages/pluggable/wrapperAdapter.ts index ad26d17f..e668c335 100644 --- a/src/storages/pluggable/wrapperAdapter.ts +++ b/src/storages/pluggable/wrapperAdapter.ts @@ -6,7 +6,7 @@ import { ICustomStorageWrapper } from '../types'; import { LOG_PREFIX } from './constants'; // Sanitizers return the given value if it is of the expected type, or a new sanitized one if invalid. -// @TODO review or remove sanitizers. Could be expensive for producer methods +// @TODO review or remove sanitizers. Could be expensive and error-prone for producer methods function sanitizeBoolean(val: any): boolean { return sBoolean(val) || false; @@ -32,9 +32,10 @@ function sanitizeNullableString(val: any): string | null { sanitizeNullableString.type = 'string | null'; function sanitizeArrayOfNullableStrings(val: any): (string | null)[] { + const isStringOrNull = (v: any) => v === null || isString(v); if (!Array.isArray(val)) return []; - if (val.every(v => v === null || isString(v))) return val; - return val.map(v => v !== null ? toString(v) : null); // otherwise, return a new array with items sanitized + if (val.every(isStringOrNull)) return val; + return val.filter(isStringOrNull); // otherwise, return a new array with items sanitized } sanitizeArrayOfNullableStrings.type = 'Array'; @@ -48,14 +49,12 @@ const METHODS_TO_PROMISE_WRAP: [string, undefined | { (val: any): any, type: str ['incr', sanitizeBoolean], ['decr', sanitizeBoolean], ['getMany', sanitizeArrayOfNullableStrings], - ['connect', sanitizeBoolean], - ['close', undefined], - - ['pushItems', undefined], ['popItems', sanitizeArrayOfStrings], ['getItemsCount', sanitizeNumber], ['itemContains', sanitizeBoolean], + ['connect', sanitizeBoolean], + ['close', undefined], ]; /** From 927d50535310884c741393dc7f602919ab17e718 Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Fri, 30 Apr 2021 10:45:23 -0300 Subject: [PATCH 14/14] removed sanitizer logic for a separete PR --- src/evaluator/value/sanitize.ts | 2 +- .../__tests__/wrapperAdapter.spec.ts | 50 ------------ src/storages/pluggable/wrapperAdapter.ts | 80 +++++-------------- 3 files changed, 19 insertions(+), 113 deletions(-) diff --git a/src/evaluator/value/sanitize.ts b/src/evaluator/value/sanitize.ts index b42ff769..991ed8e1 100644 --- a/src/evaluator/value/sanitize.ts +++ b/src/evaluator/value/sanitize.ts @@ -28,7 +28,7 @@ function sanitizeArray(val: any): string[] | undefined { return arr.length ? arr : undefined; } -export function sanitizeBoolean(val: any): boolean | undefined { +function sanitizeBoolean(val: any): boolean | undefined { if (val === true || val === false) return val; if (typeof val === 'string') { diff --git a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts index 49ad448d..d8f62ecc 100644 --- a/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts +++ b/src/storages/pluggable/__tests__/wrapperAdapter.spec.ts @@ -38,38 +38,6 @@ export const wrapperWithIssues = { itemContains: throwsException, }; -export const wrapperWithValuesToSanitize = { - get: () => Promise.resolve(10), - set: () => Promise.resolve('tRue'), - del: () => Promise.resolve(), // no result - getKeysByPrefix: () => Promise.resolve(['1', null, false, true, '2', null]), - incr: () => Promise.resolve('1'), - decr: () => Promise.resolve('0'), - getMany: () => Promise.resolve(['1', null, false, true, '2', null]), - connect: () => Promise.resolve(1), - getAndSet: () => Promise.resolve(true), - getByPrefix: () => Promise.resolve(['1', null, false, true, '2', null]), - popItems: () => Promise.resolve('invalid array'), - getItemsCount: () => Promise.resolve('10'), - itemContains: () => Promise.resolve('true'), -}; - -const SANITIZED_RESULTS = { - get: '10', - set: true, - del: false, - getKeysByPrefix: ['1', '2'], - incr: false, - decr: false, - getMany: ['1', null, '2', null], - connect: false, - getAndSet: 'true', - getByPrefix: ['1', '2'], - popItems: [], - getItemsCount: 10, - itemContains: true, -}; - const VALID_METHOD_CALLS = { 'get': ['some_key'], 'set': ['some_key', 'some_value'], @@ -136,22 +104,4 @@ describe('Wrapper Adapter', () => { expect(loggerMock.error).toBeCalledTimes(methods.length); }); - test('sanitize wrapper call results', async () => { - const instance = wrapperAdapter(loggerMock, wrapperWithValuesToSanitize); - const methods = Object.keys(SANITIZED_RESULTS); - - for (let i = 0; i < methods.length; i++) { - try { - const method = methods[i]; - const result = await instance[method](...VALID_METHOD_CALLS[method]); - - expect(result).toEqual(SANITIZED_RESULTS[method]); // result should be sanitized - } catch (e) { - expect(true).toBe(methods[i]); // promise shouldn't be rejected - } - } - - expect(loggerMock.warn).toBeCalledTimes(methods.length); // one warning for each wrapper operation call which result had to be sanitized - }); - }); diff --git a/src/storages/pluggable/wrapperAdapter.ts b/src/storages/pluggable/wrapperAdapter.ts index e668c335..06bcab0f 100644 --- a/src/storages/pluggable/wrapperAdapter.ts +++ b/src/storages/pluggable/wrapperAdapter.ts @@ -1,60 +1,24 @@ -import { toString, isString, toNumber } from '../../utils/lang'; -import { sanitizeBoolean as sBoolean } from '../../evaluator/value/sanitize'; import { ILogger } from '../../logger/types'; import { SplitError } from '../../utils/lang/errors'; import { ICustomStorageWrapper } from '../types'; import { LOG_PREFIX } from './constants'; -// Sanitizers return the given value if it is of the expected type, or a new sanitized one if invalid. -// @TODO review or remove sanitizers. Could be expensive and error-prone for producer methods - -function sanitizeBoolean(val: any): boolean { - return sBoolean(val) || false; -} -sanitizeBoolean.type = 'boolean'; - -function sanitizeNumber(val: any): number { - return toNumber(val); -} -sanitizeNumber.type = 'number'; - -function sanitizeArrayOfStrings(val: any): string[] { - if (!Array.isArray(val)) return []; // if not an array, returns a new empty one - if (val.every(isString)) return val; // if all items are valid, return the given array - return val.filter(isString); // otherwise, return a new array filtering the invalid items -} -sanitizeArrayOfStrings.type = 'Array'; - -function sanitizeNullableString(val: any): string | null { - if (val === null) return val; - return toString(val); // if not null, sanitize value as an string -} -sanitizeNullableString.type = 'string | null'; - -function sanitizeArrayOfNullableStrings(val: any): (string | null)[] { - const isStringOrNull = (v: any) => v === null || isString(v); - if (!Array.isArray(val)) return []; - if (val.every(isStringOrNull)) return val; - return val.filter(isStringOrNull); // otherwise, return a new array with items sanitized -} -sanitizeArrayOfNullableStrings.type = 'Array'; - -const METHODS_TO_PROMISE_WRAP: [string, undefined | { (val: any): any, type: string }][] = [ - ['get', sanitizeNullableString], - ['set', sanitizeBoolean], - ['getAndSet', sanitizeNullableString], - ['del', sanitizeBoolean], - ['getKeysByPrefix', sanitizeArrayOfStrings], - ['getByPrefix', sanitizeArrayOfStrings], - ['incr', sanitizeBoolean], - ['decr', sanitizeBoolean], - ['getMany', sanitizeArrayOfNullableStrings], - ['pushItems', undefined], - ['popItems', sanitizeArrayOfStrings], - ['getItemsCount', sanitizeNumber], - ['itemContains', sanitizeBoolean], - ['connect', sanitizeBoolean], - ['close', undefined], +const METHODS_TO_PROMISE_WRAP: string[] = [ + 'get', + 'set', + 'getAndSet', + 'del', + 'getKeysByPrefix', + 'getByPrefix', + 'incr', + 'decr', + 'getMany', + 'pushItems', + 'popItems', + 'getItemsCount', + 'itemContains', + 'connect', + 'close' ]; /** @@ -69,7 +33,7 @@ export function wrapperAdapter(log: ILogger, wrapper: ICustomStorageWrapper): IC const wrapperAdapter: Record = {}; - METHODS_TO_PROMISE_WRAP.forEach(([method, sanitizer]) => { + METHODS_TO_PROMISE_WRAP.forEach((method) => { // Logs error and wraps it into an SplitError object (required to handle user callback errors in SDK readiness events) function handleError(e: any) { @@ -80,15 +44,7 @@ export function wrapperAdapter(log: ILogger, wrapper: ICustomStorageWrapper): IC wrapperAdapter[method] = function () { try { // @ts-ignore - return wrapper[method].apply(wrapper, arguments).then((value => { - if (!sanitizer) return value; - const sanitizedValue = sanitizer(value); - - // if value had to be sanitized, log a warning - if (sanitizedValue !== value) log.warn(`${LOG_PREFIX} Attempted to sanitize return value [${value}] of wrapper '${method}' operation which should be of type [${sanitizer.type}]. Sanitized and processed value => [${sanitizedValue}]`); - - return sanitizedValue; - })).catch(handleError); + return wrapper[method].apply(wrapper, arguments).then(value => value).catch(handleError); } catch (e) { return handleError(e); }