From e6064ece1b29033539f524afecdea6b83327d0ff Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Tue, 18 Jan 2022 12:41:06 -0300 Subject: [PATCH 1/2] remove old telemetry modules --- src/storages/inMemory/CountsCacheInMemory.ts | 37 -------------- .../inMemory/LatenciesCacheInMemory.ts | 45 ----------------- src/storages/inRedis/CountsCacheInRedis.ts | 20 -------- src/storages/inRedis/LatenciesCacheInRedis.ts | 23 --------- src/sync/submitters/metricsSyncTask.ts | 49 ------------------- 5 files changed, 174 deletions(-) delete mode 100644 src/storages/inMemory/CountsCacheInMemory.ts delete mode 100644 src/storages/inMemory/LatenciesCacheInMemory.ts delete mode 100644 src/storages/inRedis/CountsCacheInRedis.ts delete mode 100644 src/storages/inRedis/LatenciesCacheInRedis.ts delete mode 100644 src/sync/submitters/metricsSyncTask.ts diff --git a/src/storages/inMemory/CountsCacheInMemory.ts b/src/storages/inMemory/CountsCacheInMemory.ts deleted file mode 100644 index ff7cd591..00000000 --- a/src/storages/inMemory/CountsCacheInMemory.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { ICountsCacheSync } from '../types'; - -export class CountsCacheInMemory implements ICountsCacheSync { - - private counters: Record = {}; - - /** - * Add counts. - */ - track(metricName: string) { - if (this.counters[metricName] === undefined) this.counters[metricName] = 1; - else this.counters[metricName]++; - - return true; - } - - /** - * Clear the collector - */ - clear() { - this.counters = {}; - } - - /** - * Get the collected data, used as payload for posting. - */ - state() { - return this.counters; - } - - /** - * Check if the cache is empty. - */ - isEmpty() { - return Object.keys(this.counters).length === 0; - } -} diff --git a/src/storages/inMemory/LatenciesCacheInMemory.ts b/src/storages/inMemory/LatenciesCacheInMemory.ts deleted file mode 100644 index 240b5a8b..00000000 --- a/src/storages/inMemory/LatenciesCacheInMemory.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { ILatenciesCacheSync } from '../types'; -import { findLatencyIndex } from '../findLatencyIndex'; - -export class LatenciesCacheInMemory implements ILatenciesCacheSync { - - private counters: Record = {}; - - /** - * Add latencies. - */ - track(metricName: string, latency: number) { - // Initialize if needed - if (this.counters[metricName] === undefined) { - this.counters[metricName] = [ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - ]; - } - - // +1 based on the latency number - this.counters[metricName][findLatencyIndex(latency)]++; - - return true; - } - - /** - * Clear the collector - */ - clear() { - this.counters = {}; - } - - /** - * Get the collected data, used as payload for posting. - */ - state() { - return this.counters; - } - - /** - * Check if the cache is empty. - */ - isEmpty() { - return Object.keys(this.counters).length === 0; - } -} diff --git a/src/storages/inRedis/CountsCacheInRedis.ts b/src/storages/inRedis/CountsCacheInRedis.ts deleted file mode 100644 index cb551fa2..00000000 --- a/src/storages/inRedis/CountsCacheInRedis.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { ICountsCacheAsync } from '../types'; -import { KeyBuilderSS } from '../KeyBuilderSS'; -import { Redis } from 'ioredis'; - -export class CountsCacheInRedis implements ICountsCacheAsync { - - private readonly redis: Redis; - private readonly keys: KeyBuilderSS; - - constructor(keys: KeyBuilderSS, redis: Redis) { - this.keys = keys; - this.redis = redis; - } - - track(metricName: string): Promise { - return this.redis.incr(this.keys.buildCountKey(metricName)).catch(() => { - // noop, for telemetry metrics there's no need to throw. - }).then(() => true); - } -} diff --git a/src/storages/inRedis/LatenciesCacheInRedis.ts b/src/storages/inRedis/LatenciesCacheInRedis.ts deleted file mode 100644 index f69b7244..00000000 --- a/src/storages/inRedis/LatenciesCacheInRedis.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { ILatenciesCacheAsync } from '../types'; -import { KeyBuilderSS } from '../KeyBuilderSS'; -import { findLatencyIndex } from '../findLatencyIndex'; -import { Redis } from 'ioredis'; - -export class LatenciesCacheInRedis implements ILatenciesCacheAsync { - - private readonly redis: Redis; - private readonly keys: KeyBuilderSS; - - constructor(keys: KeyBuilderSS, redis: Redis) { - this.keys = keys; - this.redis = redis; - } - - track(metricName: string, latency: number): Promise { - const bucketNumber = findLatencyIndex(latency); - - return this.redis.incr(this.keys.buildLatencyKey(metricName, bucketNumber)).catch(() => { - // noop, for telemetry metrics there's no need to throw. - }).then(() => true); - } -} diff --git a/src/sync/submitters/metricsSyncTask.ts b/src/sync/submitters/metricsSyncTask.ts deleted file mode 100644 index a5264b14..00000000 --- a/src/sync/submitters/metricsSyncTask.ts +++ /dev/null @@ -1,49 +0,0 @@ - -import { forOwn } from '../../utils/lang'; -import { ICountsCacheSync, ILatenciesCacheSync } from '../../storages/types'; -import { IPostMetricsCounters, IPostMetricsTimes } from '../../services/types'; -import { ISyncTask, ITimeTracker } from '../types'; -import { submitterSyncTaskFactory } from './submitterSyncTask'; -import { ILogger } from '../../logger/types'; - -// extract POST payload object from cache -function fromCache(propertyName: 'latencies' | 'delta') { - return (data: Record): any[] => { - const result: any[] = []; - - forOwn(data, (value, key) => { - result.push({ name: key, [propertyName]: value }); - }); - - return result; - }; -} - -/** - * Sync task that periodically posts telemetry counts - */ -export function countsSyncTaskFactory( - log: ILogger, - postMetricsCounters: IPostMetricsCounters, - countsCache: ICountsCacheSync, - metricsRefreshRate: number, - latencyTracker?: ITimeTracker -): ISyncTask { - - return submitterSyncTaskFactory(log, postMetricsCounters, countsCache, metricsRefreshRate, 'count metrics', latencyTracker, fromCache('delta')); -} - -/** - * Sync task that periodically posts telemetry latencies - */ -export function latenciesSyncTaskFactory( - log: ILogger, - postMetricsLatencies: IPostMetricsTimes, - latenciesCache: ILatenciesCacheSync, - metricsRefreshRate: number, - latencyTracker?: ITimeTracker -): ISyncTask { - - // don't retry metrics. - return submitterSyncTaskFactory(log, postMetricsLatencies, latenciesCache, metricsRefreshRate, 'latency metrics', latencyTracker, fromCache('latencies'), 0, true); -} From a8413da67d2908705c2e959f97dfeb78c2c23cda Mon Sep 17 00:00:00 2001 From: Emiliano Sanchez Date: Wed, 19 Jan 2022 11:21:04 -0300 Subject: [PATCH 2/2] remove old telemetry cache tests --- .../__tests__/CountsCacheInMemory.spec.ts | 14 ------- .../__tests__/LatenciesCacheInMemory.spec.ts | 34 ---------------- .../__tests__/CountsCacheInRedis.spec.ts | 35 ----------------- .../__tests__/LatenciesCacheInRedis.spec.ts | 39 ------------------- src/storages/inRedis/index.ts | 5 +-- 5 files changed, 1 insertion(+), 126 deletions(-) delete mode 100644 src/storages/inMemory/__tests__/CountsCacheInMemory.spec.ts delete mode 100644 src/storages/inMemory/__tests__/LatenciesCacheInMemory.spec.ts delete mode 100644 src/storages/inRedis/__tests__/CountsCacheInRedis.spec.ts delete mode 100644 src/storages/inRedis/__tests__/LatenciesCacheInRedis.spec.ts diff --git a/src/storages/inMemory/__tests__/CountsCacheInMemory.spec.ts b/src/storages/inMemory/__tests__/CountsCacheInMemory.spec.ts deleted file mode 100644 index bb8ebcc0..00000000 --- a/src/storages/inMemory/__tests__/CountsCacheInMemory.spec.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { CountsCacheInMemory } from '../CountsCacheInMemory'; - -test('COUNT CACHE IN MEMORY / should count metric names incrementatly', () => { - const cache = new CountsCacheInMemory(); - - cache.track('counted-metric-one'); - cache.track('counted-metric-one'); - cache.track('counted-metric-two'); - - const state = cache.state(); - - expect(state['counted-metric-one']).toBe(2); - expect(state['counted-metric-two']).toBe(1); -}); diff --git a/src/storages/inMemory/__tests__/LatenciesCacheInMemory.spec.ts b/src/storages/inMemory/__tests__/LatenciesCacheInMemory.spec.ts deleted file mode 100644 index 18ad0a3a..00000000 --- a/src/storages/inMemory/__tests__/LatenciesCacheInMemory.spec.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { LatenciesCacheInMemory } from '../LatenciesCacheInMemory'; - -test('METRICS (LATENCIES) CACHE IN MEMORY / should count based on ranges', () => { - const metricName = 'testing'; - const c1 = new LatenciesCacheInMemory(); - - c1.track(metricName, 1); - c1.track(metricName, 1.2); - c1.track(metricName, 1.4); - - expect(c1.state()[metricName][0] === 3).toBe(true); // the bucket #0 should have 3 - - c1.track(metricName, 1.5); - expect(c1.state()[metricName][1] === 1).toBe(true); // the bucket #1 should have 1 - - c1.track(metricName, 2.25); - c1.track(metricName, 2.26); - c1.track(metricName, 2.265); - expect(c1.state()[metricName][2] === 3).toBe(true); // the bucket #3 should have 1 - - c1.track(metricName, 985251); - expect(c1.state()[metricName][23] === 1).toBe(true); // the bucket #23 should have 1 - -}); - -test('METRICS (LATENCIES) CACHE IN MEMORY / clear', () => { - const metricName = 'testing'; - const c1 = new LatenciesCacheInMemory(); - - c1.track(metricName, 1); - c1.track(metricName, 1000); - c1.clear(); - expect(c1.isEmpty()).toBe(true); // after call clear, the cache should be empty -}); diff --git a/src/storages/inRedis/__tests__/CountsCacheInRedis.spec.ts b/src/storages/inRedis/__tests__/CountsCacheInRedis.spec.ts deleted file mode 100644 index b8032ac2..00000000 --- a/src/storages/inRedis/__tests__/CountsCacheInRedis.spec.ts +++ /dev/null @@ -1,35 +0,0 @@ -import Redis from 'ioredis'; -import { KeyBuilderSS } from '../../KeyBuilderSS'; -import { CountsCacheInRedis } from '../CountsCacheInRedis'; - -const prefix = 'counts_cache_ut'; -const metadata = { s: 'version', i: 'ip', n: 'hostname' }; - -test('COUNTS CACHE IN REDIS / cover basic behavior', async () => { - const connection = new Redis({}); - const keys = new KeyBuilderSS(prefix, metadata); - const cache = new CountsCacheInRedis(keys, connection); - - await cache.track('counted-metric-one'); - await cache.track('counted-metric-one'); - - const keyOne = keys.buildCountKey('counted-metric-one'); - const keyTwo = keys.buildCountKey('counted-metric-two'); - - let metricOneValue = await connection.get(keyOne); - expect(metricOneValue).toBe('2'); - - await cache.track('counted-metric-two'); - - metricOneValue = await connection.get(keyOne); - let metricTwoValue = await connection.get(keyTwo); - - expect(metricOneValue).toBe('2'); - expect(metricTwoValue).toBe('1'); - - // Clean up - const keysToClean = await connection.keys(`${prefix}*`); - if (keysToClean.length) await connection.del(keysToClean); - - await connection.quit(); -}); diff --git a/src/storages/inRedis/__tests__/LatenciesCacheInRedis.spec.ts b/src/storages/inRedis/__tests__/LatenciesCacheInRedis.spec.ts deleted file mode 100644 index 7567b969..00000000 --- a/src/storages/inRedis/__tests__/LatenciesCacheInRedis.spec.ts +++ /dev/null @@ -1,39 +0,0 @@ -import Redis from 'ioredis'; -import { KeyBuilderSS } from '../../KeyBuilderSS'; -import { LatenciesCacheInRedis } from '../LatenciesCacheInRedis'; - -const prefix = 'latencies_cache_UT'; -const metadata = { s: 'js_someversion', i: 'some_ip', n: 'some_hostname' }; - -test('METRICS (LATENCIES) CACHE IN REDIS / should count based on ranges', async () => { - const connection = new Redis(); - const keys = new KeyBuilderSS(prefix, metadata); - const cache = new LatenciesCacheInRedis(keys, connection); - const metricName = 'testing'; - - await cache.track(metricName, 1); - await cache.track(metricName, 1.2); - await cache.track(metricName, 1.4); - - expect(await connection.get(keys.buildLatencyKey(metricName, 0))).toBe('3'); // the bucket #0 should have 3 - - await cache.track(metricName, 1.5); - - expect(await connection.get(keys.buildLatencyKey(metricName, 1))).toBe('1'); // the bucket #1 should have 1 - - await cache.track(metricName, 2.25); - await cache.track(metricName, 2.26); - await cache.track(metricName, 2.265); - - expect(await connection.get(keys.buildLatencyKey(metricName, 2))).toBe('3'); // the bucket #2 should have 3 - - await cache.track(metricName, 985251); - - expect(await connection.get(keys.buildLatencyKey(metricName, 23))).toBe('1'); // the bucket #23 should have 1 - - // Clean up post-test - const keysToClean = await connection.keys(`${prefix}*`); - await connection.del(keysToClean); - - await connection.quit(); -}); diff --git a/src/storages/inRedis/index.ts b/src/storages/inRedis/index.ts index de72a220..c22ee15d 100644 --- a/src/storages/inRedis/index.ts +++ b/src/storages/inRedis/index.ts @@ -6,8 +6,6 @@ import { SplitsCacheInRedis } from './SplitsCacheInRedis'; import { SegmentsCacheInRedis } from './SegmentsCacheInRedis'; import { ImpressionsCacheInRedis } from './ImpressionsCacheInRedis'; import { EventsCacheInRedis } from './EventsCacheInRedis'; -import { LatenciesCacheInRedis } from './LatenciesCacheInRedis'; -import { CountsCacheInRedis } from './CountsCacheInRedis'; import { STORAGE_REDIS } from '../../utils/constants'; export interface InRedisStorageOptions { @@ -38,8 +36,7 @@ export function InRedisStorage(options: InRedisStorageOptions = {}): IStorageAsy segments: new SegmentsCacheInRedis(log, keys, redisClient), impressions: new ImpressionsCacheInRedis(log, keys.buildImpressionsKey(), redisClient, metadata), events: new EventsCacheInRedis(log, keys.buildEventsKey(), redisClient, metadata), - latencies: new LatenciesCacheInRedis(keys, redisClient), - counts: new CountsCacheInRedis(keys, redisClient), + // @TODO add new telemetry cache // When using REDIS we should: // 1- Disconnect from the storage