Start work to make the deduplication plugin work like the .NET one

Author: srijken

dist/exceptionless.d.ts

@@ -8,6 +8,7 @@ export interface IEvent {
     value?: number;
     data?: any;
     reference_id?: string;
+    count?: number;
 }
 export declare class SubmissionResponse {
     success: boolean;
@@ -428,8 +429,10 @@ export declare class DuplicateCheckerPlugin implements IEventPlugin {
     name: string;
     private _processedHashcodes;
     private _getCurrentTime;
-    constructor(getCurrentTime?: () => number);
+    constructor(getCurrentTime?: () => number, interval?: number);
     run(context: EventPluginContext, next?: () => void): void;
+    private onInterval();
+    private enqueueMergedEvents();
 }
 export declare class EventExclusionPlugin implements IEventPlugin {
     priority: number;

dist/exceptionless.js

@@ -8,4 +8,5 @@ export interface IEvent {
   value?: number;
   data?: any;
   reference_id?: string;
+  count?: number;
 }

dist/exceptionless.js.map

@@ -1,4 +1,7 @@
 import { ContextData } from '../ContextData';
+import { DefaultErrorParser } from '../../services/DefaultErrorParser';
+import { ExceptionlessClient } from '../../ExceptionlessClient';
+import { IEvent } from '../../models/IEvent';
 import { EventPluginContext } from '../EventPluginContext';
 import { DuplicateCheckerPlugin } from './DuplicateCheckerPlugin';
 import { ErrorPlugin } from './ErrorPlugin';
@@ -7,33 +10,39 @@ import { expect } from 'chai';
 
 describe('DuplicateCheckerPlugin', () => {
   let now: number = 0;
+  let client: ExceptionlessClient;
   let plugin: DuplicateCheckerPlugin;
 
   beforeEach(() => {
-    plugin = new DuplicateCheckerPlugin(() => now);
+    client = new ExceptionlessClient('LhhP1C9gijpSKCslHHCvwdSIz298twx271n1l6xw', 'http://localhost:50000');
+    plugin = new DuplicateCheckerPlugin(() => now, 50);
   });
 
   function run(exception: Error) {
-    let context: EventPluginContext;
-    let contextData: ContextData;
-    ({ context, contextData } = createFixture());
+    let errorParser = new DefaultErrorParser();
+    let context = new EventPluginContext(client, { type: 'error', data: {} });
+    context.event.data['@error'] = errorParser.parse(context, exception);
 
-    contextData.setException(exception);
-
-    let errorPlugin = new ErrorPlugin();
-    errorPlugin.run(context);
     plugin.run(context);
 
     return context;
   }
 
-  it('should ignore duplicate within window', () => {
+  it('should ignore duplicate within window', (done) => {
     let exception = createException([{
       name: 'methodA'
     }]);
     run(exception);
+
     let contextOfSecondRun = run(exception);
     expect(contextOfSecondRun.cancelled).to.be.true;
+    setTimeout(() => {
+
+      expect(contextOfSecondRun.event.count).to.equal(1);
+
+      done();
+      }, 100);
+
   });
 
   it('shouldn\'t ignore error without stack', () => {
@@ -68,8 +77,11 @@ describe('DuplicateCheckerPlugin', () => {
 });
 
 function createException(stack?) {
+  function throwError() {
+    throw new ReferenceError('This is a test');
+  }
   try {
-    throw new Error();
+    throwError();
   } catch (e) {
     e.testStack = stack;
     return e;

dist/exceptionless.min.js

@@ -5,46 +5,50 @@ import { EventPluginContext } from '../EventPluginContext';
 import { Utils } from '../../Utils';
 
 export class DuplicateCheckerPlugin implements IEventPlugin {
-  public priority: number = 40;
+  public priority: number = 90;
   public name: string = 'DuplicateCheckerPlugin';
 
+  private _mergedEvents: MergedEvent[] = [];
   private _processedHashcodes: TimestampedHash[] = [];
   private _getCurrentTime: () => number;
+  private _interval: number;
 
-  constructor(getCurrentTime: () => number = () => Date.now()) {
+  constructor(getCurrentTime: () => number = () => Date.now(), interval: number = 60000) {
     this._getCurrentTime = getCurrentTime;
+    this._interval = interval;
+
+    setInterval(() => {
+      while(this._mergedEvents.length > 0){
+        this._mergedEvents.shift().resubmit();
+      }
+    }, interval);
   }
 
   public run(context: EventPluginContext, next?: () => void): void {
-    function isDuplicate(error: IInnerError, processedHashcodes, now, log: ILog): boolean {
-      while (error) {
-        let hashCode = Utils.getHashCode(error.stack_trace && JSON.stringify(error.stack_trace));
-
-        // Only process the unique errors times within a 2 second window.
-        if (hashCode && processedHashcodes.some(h => h.hash === hashCode && h.timestamp >= (now - 2000))) {
-          log.info(`Ignoring duplicate error event hash: ${hashCode}`);
-          return true;
-        }
+    let hashCode = Utils.getHashCode(JSON.stringify(context.event.data['@error'], ['stack_trace', 'inner']));
+    let count = context.event.count || 1;
 
-        // Add this exception to our list of recent processed errors.
-        processedHashcodes.push({ hash: hashCode, timestamp: now });
+    let now = this._getCurrentTime();
 
-        // Only keep the last 20 recent errors.
-        while (processedHashcodes.length > 20) {
-          processedHashcodes.shift();
-        }
-
-        error = error.inner;
-      }
+    let merged = this._mergedEvents.filter(s => s.hashCode === hashCode)[0];
+    if(merged) {
+      merged.incrementCount(count);
+      merged.updateDate(context.event.date);
+      context.cancelled = true;
+      return;
+    }
 
-      return false;
+    if(this._processedHashcodes.some(h => h.hash === hashCode && h.timestamp >= (now - this._interval))) {
+      this._mergedEvents.push(new MergedEvent(hashCode, context, count));
+      context.cancelled = true;
+      return;
     }
 
-    if (context.event.type === 'error') {
-      if (isDuplicate(context.event.data['@error'], this._processedHashcodes, this._getCurrentTime(), context.log)) {
-        context.cancelled = true;
-        return;
-      }
+    this._processedHashcodes.push({ hash: hashCode, timestamp: now });
+
+    // Only keep the last 50 recent errors.
+    while (this._processedHashcodes.length > 50) {
+      this._processedHashcodes.shift();
     }
 
     next && next();
@@ -55,3 +59,30 @@ interface TimestampedHash {
   hash: number;
   timestamp: number;
 }
+
+class MergedEvent {
+  private _count: number;
+  private _context: EventPluginContext;
+  public hashCode: number;
+
+  constructor(hashCode: number, context: EventPluginContext, count: number){
+    this.hashCode = hashCode;
+    this._context = context;
+    this._count = count;
+  }
+
+  public incrementCount(count: number){
+    this._count += count;
+  }
+
+  public resubmit() {
+    this._context.event.count = this._count;
+    this._context.client.config.queue.enqueue(this._context.event);
+  }
+
+  public updateDate(date) {
+    if(date > this._context.event.date) {
+      this._context.event.date = date;
+    }
+  }
+}