Examples with Stat com.nextdoor.bender.monitoring.Stat used on opensource projects

Example 1 with Stat

use of com.nextdoor.bender.monitoring.Stat in project bender by Nextdoor.

the class BaseHandler method processInternal.

/**
 * Method called by Handler implementations to process records.
 *
 * @param context Lambda invocation context.
 * @throws HandlerException
 */
private void processInternal(Context context) throws HandlerException {
    Stat runtime = new Stat("runtime.ns");
    runtime.start();
    Source source = this.getSource();
    DeserializerProcessor deser = source.getDeserProcessor();
    List<OperationProcessor> operations = source.getOperationProcessors();
    List<String> containsStrings = source.getContainsStrings();
    List<Pattern> regexPatterns = source.getRegexPatterns();
    this.getIpcService().setContext(context);
    Iterator<InternalEvent> events = this.getInternalEventIterator();
    /*
     * For logging purposes log when the function started running
     */
    this.monitor.invokeTimeNow();
    AtomicLong eventCount = new AtomicLong(0);
    AtomicLong oldestArrivalTime = new AtomicLong(System.currentTimeMillis());
    AtomicLong oldestOccurrenceTime = new AtomicLong(System.currentTimeMillis());
    /*
     * Process each record
     */
    int characteristics = Spliterator.IMMUTABLE;
    Spliterator<InternalEvent> spliterator = Spliterators.spliteratorUnknownSize(events, characteristics);
    Stream<InternalEvent> input = StreamSupport.stream(spliterator, false);
    /*
     * Filter out raw events
     */
    Stream<InternalEvent> filtered = input.filter(/*
         * Perform regex filter
         */
    ievent -> {
        eventCount.incrementAndGet();
        String eventStr = ievent.getEventString();
        /*
           * Apply String contains filters before deserialization
           */
        for (String containsString : containsStrings) {
            if (eventStr.contains(containsString)) {
                return false;
            }
        }
        /*
           * Apply regex patterns before deserialization
           */
        for (Pattern regexPattern : regexPatterns) {
            Matcher m = regexPattern.matcher(eventStr);
            if (m.find()) {
                return false;
            }
        }
        return true;
    });
    /*
     * Deserialize
     */
    Stream<InternalEvent> deserialized = filtered.map(ievent -> {
        DeserializedEvent data = deser.deserialize(ievent.getEventString());
        if (data == null || data.getPayload() == null) {
            logger.warn("Failed to deserialize: " + ievent.getEventString());
            return null;
        }
        ievent.setEventObj(data);
        return ievent;
    }).filter(Objects::nonNull);
    /*
     * Perform Operations
     */
    Stream<InternalEvent> operated = deserialized;
    for (OperationProcessor operation : operations) {
        operated = operation.perform(operated);
    }
    /*
     * Serialize
     */
    Stream<InternalEvent> serialized = operated.map(ievent -> {
        try {
            String raw = null;
            raw = this.ser.serialize(this.wrapper.getWrapped(ievent));
            ievent.setSerialized(raw);
            return ievent;
        } catch (SerializationException e) {
            return null;
        }
    }).filter(Objects::nonNull);
    /*
     * Transport
     */
    serialized.forEach(ievent -> {
        /*
       * Update times
       */
        updateOldest(oldestArrivalTime, ievent.getArrivalTime());
        updateOldest(oldestOccurrenceTime, ievent.getEventTime());
        try {
            this.getIpcService().add(ievent);
        } catch (TransportException e) {
            logger.warn("error adding event", e);
        }
    });
    /*
     * Wait for transporters to finish
     */
    try {
        this.getIpcService().shutdown();
    } catch (TransportException e) {
        throw new HandlerException("encounted TransportException while shutting down ipcService", e);
    } catch (InterruptedException e) {
        throw new HandlerException("thread was interruptedwhile shutting down ipcService", e);
    } finally {
        String evtSource = this.getSourceName();
        runtime.stop();
        if (!this.skipWriteStats) {
            writeStats(eventCount.get(), oldestArrivalTime.get(), oldestOccurrenceTime.get(), evtSource, runtime);
        }
        if (logger.isTraceEnabled()) {
            getGCStats();
        }
    }
}

Example 2 with Stat

use of com.nextdoor.bender.monitoring.Stat in project bender by Nextdoor.

the class DeserializerProcessorTest method testOnUnexpectedError.

@Test(expected = RuntimeException.class)
public void testOnUnexpectedError() {
    DummyDeserializer mockDeser = mock(DummyDeserializer.class);
    when(mockDeser.deserialize("foo")).thenThrow(new RuntimeException("unexpected"));
    DeserializerProcessor deser = new DeserializerProcessor(mockDeser);
    /*
     * Mock the Stat object
     */
    Stat runtimeStat = mock(Stat.class);
    Stat successStat = mock(Stat.class);
    Stat errorStat = mock(Stat.class);
    deser.setRuntimeStat(runtimeStat);
    deser.setSuccessCountStat(successStat);
    deser.setErrorCountStat(errorStat);
    try {
        deser.deserialize("foo");
    } catch (DeserializationException e) {
    // expected
    }
}

Example 3 with Stat

use of com.nextdoor.bender.monitoring.Stat in project bender by Nextdoor.

the class OperationProcessorTest method testStatsLoggingOnError.

@Test
public void testStatsLoggingOnError() {
    DummyOperation operation = mock(DummyOperation.class);
    DummyOperationFactory mutatorFactory = new DummyOperationFactory(operation);
    OperationProcessor processor = new OperationProcessor(mutatorFactory);
    InternalEvent ievent = new InternalEvent("a", null, 1);
    doThrow(new OperationException("Expceted")).when(operation).perform(ievent);
    /*
     * Mock the Stat object
     */
    Stat runtimeStat = mock(Stat.class);
    Stat successStat = mock(Stat.class);
    Stat errorStat = mock(Stat.class);
    processor.setRuntimeStat(runtimeStat);
    processor.setSuccessCountStat(successStat);
    processor.setErrorCountStat(errorStat);
    Stream<InternalEvent> stream = processor.perform(Stream.of(ievent));
    List<InternalEvent> output = stream.collect(Collectors.toList());
    /*
     * Verify start, stop are called, increment error count and never increment success count.
     */
    verify(runtimeStat, times(1)).start();
    verify(runtimeStat, times(1)).stop();
    verify(successStat, never()).increment();
    verify(errorStat, times(1)).increment();
    /*
     * Verify contents of output stream
     */
    assertEquals(0, output.size());
}

Example 4 with Stat

use of com.nextdoor.bender.monitoring.Stat in project bender by Nextdoor.

the class IpcSenderServiceTest method testStatsLoggingOnError.

@Test
public void testStatsLoggingOnError() throws InstantiationException, IllegalAccessException, InterruptedException, TransportException {
    DummyTransporter mockDummyTransporter = mock(DummyTransporter.class);
    DummyTransporterFactory tfactory = new DummyTransporterFactory();
    tfactory.transporter = mockDummyTransporter;
    doThrow(new TransportException("expected exception in test")).when(mockDummyTransporter).sendBatch(any(DummyTransportBuffer.class));
    IpcSenderService ipc = new IpcSenderService(tfactory);
    /*
     * Mock the Stat object
     */
    Stat successStat = mock(Stat.class);
    Stat errorStat = mock(Stat.class);
    ipc.setSuccessCountStat(successStat);
    ipc.setErrorCountStat(errorStat);
    /*
     * Every 5 adds a send should happen.
     */
    for (int i = 0; i < 5; i++) {
        ipc.add(mock(InternalEvent.class));
    }
    try {
        ipc.shutdown();
    } catch (TransportException e) {
    // expected
    }
    /*
     * The sendBatch method will be called twice and each will throw an error. Verify that error
     * counting happens as expected.
     */
    verify(successStat, never()).increment();
    verify(errorStat, times(1)).increment();
}

Example 5 with Stat

use of com.nextdoor.bender.monitoring.Stat in project bender by Nextdoor.

the class SerializerProcessorTest method testStatsLoggingOnError.

@Test
public void testStatsLoggingOnError() {
    DummySerializer serializer = mock(DummySerializer.class);
    SerializerProcessor processor = new SerializerProcessor(serializer);
    doThrow(new RuntimeException()).when(serializer).serialize("foo");
    /*
     * Mock the Stat object
     */
    Stat runtimeStat = mock(Stat.class);
    Stat successStat = mock(Stat.class);
    Stat errorStat = mock(Stat.class);
    processor.setRuntimeStat(runtimeStat);
    processor.setSuccessCountStat(successStat);
    processor.setErrorCountStat(errorStat);
    try {
        processor.serialize("foo");
    } catch (Exception e) {
    // expected
    }
    /*
     * Verify start, stop are called, increment error count and never increment success count.
     */
    verify(runtimeStat, times(1)).start();
    verify(runtimeStat, times(1)).stop();
    verify(successStat, never()).increment();
    verify(errorStat, times(1)).increment();
}