Examples with Queue com.oath.cyclops.async.adapters.Queue used on opensource projects

Example 31 with Queue

use of com.oath.cyclops.async.adapters.Queue in project bender by Nextdoor.

the class ConditionalOperationTest method testTwoConditionsNoMatch.

@Test
public void testTwoConditionsNoMatch() {
    List<Pair<FilterOperation, List<OperationProcessor>>> conditions = new ArrayList<Pair<FilterOperation, List<OperationProcessor>>>();
    /*
     * Case 1
     */
    List<OperationProcessor> case1Ops = new ArrayList<OperationProcessor>();
    DummyAppendOperationFactory pos = new DummyAppendOperationFactory();
    DummyAppendOperationConfig posConf = new DummyAppendOperationConfig();
    posConf.setAppendStr("+");
    pos.setConf(posConf);
    case1Ops.add(new OperationProcessor(pos));
    FilterOperation case1Filter = new BasicFilterOperation(false);
    conditions.add(new ImmutablePair<FilterOperation, List<OperationProcessor>>(case1Filter, case1Ops));
    /*
     * Case 2
     */
    List<OperationProcessor> case2Ops = new ArrayList<OperationProcessor>();
    DummyAppendOperationFactory neg = new DummyAppendOperationFactory();
    DummyAppendOperationConfig negConf = new DummyAppendOperationConfig();
    negConf.setAppendStr("-");
    neg.setConf(negConf);
    case2Ops.add(new OperationProcessor(neg));
    FilterOperation case2Filter = new BasicFilterOperation(false);
    conditions.add(new ImmutablePair<FilterOperation, List<OperationProcessor>>(case2Filter, case2Ops));
    ConditionalOperation op = new ConditionalOperation(conditions, false);
    /*
     * Create thread that supplies input events
     */
    Queue<InternalEvent> inputQueue = new Queue<InternalEvent>();
    supply(2, inputQueue);
    /*
     * Process
     */
    Stream<InternalEvent> input = inputQueue.stream();
    Stream<InternalEvent> output = op.getOutputStream(input);
    List<String> actual = output.map(m -> {
        return m.getEventObj().getPayload().toString();
    }).collect(Collectors.toList());
    List<String> expected = Arrays.asList("0", "1");
    assertEquals(2, actual.size());
    assertTrue(expected.containsAll(actual));
}

Example 32 with Queue

use of com.oath.cyclops.async.adapters.Queue in project bender by Nextdoor.

the class ConditionalOperationTest method testFilterNonMatch.

public void testFilterNonMatch() {
    List<Pair<FilterOperation, List<OperationProcessor>>> conditions = new ArrayList<Pair<FilterOperation, List<OperationProcessor>>>();
    /*
     * Case 1
     */
    List<OperationProcessor> case1Ops = new ArrayList<OperationProcessor>();
    FilterOperation case1Filter = new BasicFilterOperation(false);
    conditions.add(new ImmutablePair<FilterOperation, List<OperationProcessor>>(case1Filter, case1Ops));
    ConditionalOperation op = new ConditionalOperation(conditions, true);
    /*
     * Create thread that supplies input events
     */
    Queue<InternalEvent> inputQueue = new Queue<InternalEvent>();
    supply(2, inputQueue);
    /*
     * Process
     */
    Stream<InternalEvent> input = inputQueue.stream();
    Stream<InternalEvent> output = op.getOutputStream(input);
    List<String> actual = output.map(m -> {
        return m.getEventObj().getPayload().toString();
    }).collect(Collectors.toList());
    assertEquals(0, actual.size());
}

Example 33 with Queue

use of com.oath.cyclops.async.adapters.Queue in project bender by Nextdoor.

the class ConditionalOperationTest method testFilterCondition.

@Test
public void testFilterCondition() {
    List<Pair<FilterOperation, List<OperationProcessor>>> conditions = new ArrayList<Pair<FilterOperation, List<OperationProcessor>>>();
    /*
     * Case 1
     */
    List<OperationProcessor> case1Ops = new ArrayList<OperationProcessor>();
    BasicFilterOperationFactory fOp = new BasicFilterOperationFactory();
    BasicFilterOperationConfig fOpConf = new BasicFilterOperationConfig();
    fOpConf.setPass(false);
    fOp.setConf(fOpConf);
    case1Ops.add(new OperationProcessor(fOp));
    FilterOperation case1Filter = new BasicFilterOperation(true);
    conditions.add(new ImmutablePair<FilterOperation, List<OperationProcessor>>(case1Filter, case1Ops));
    ConditionalOperation op = new ConditionalOperation(conditions, false);
    /*
     * Create thread that supplies input events
     */
    Queue<InternalEvent> inputQueue = new Queue<InternalEvent>();
    supply(2, inputQueue);
    /*
     * Process
     */
    Stream<InternalEvent> input = inputQueue.stream();
    Stream<InternalEvent> output = op.getOutputStream(input);
    List<String> actual = output.map(m -> {
        return m.getEventObj().getPayload().toString();
    }).collect(Collectors.toList());
    assertEquals(0, actual.size());
}

Example 34 with Queue

use of com.oath.cyclops.async.adapters.Queue in project bender by Nextdoor.

the class BaseHandlerQueueTest method testSlowSourceSlowConsumer.

@Test
public void testSlowSourceSlowConsumer() throws HandlerException {
    BaseHandler.CONFIG_FILE = "/config/handler_config_queue_throttle.yaml";
    Queue<DummyEvent> q = new Queue<DummyEvent>();
    Iterator<DummyEvent> dummyEvents = q.stream().iterator();
    new Thread(new Runnable() {

        @Override
        public void run() {
            for (int i = 0; i < 1000; i++) {
                if (i % 500 == 0) {
                    try {
                        Thread.sleep(1000);
                    } catch (InterruptedException e) {
                    }
                }
                q.offer(new DummyEvent("" + i, 0));
            }
            q.close();
        }
    }).start();
    TestContext context = new TestContext();
    context.setInvokedFunctionArn("arn:aws:lambda:us-east-1:123:function:test:tag");
    handler.handler(dummyEvents, context);
    /*
     * Verify Events made it all the way through
     */
    assertEquals(1000, BufferedTransporter.output.size());
}

Example 35 with Queue

use of com.oath.cyclops.async.adapters.Queue in project bender by Nextdoor.

the class BaseHandler method processInternal.

/**
 * Method called by Handler implementations to process records.
 *
 * @param context Lambda invocation context.
 * @throws HandlerException
 */
private void processInternal(Context context) throws HandlerException {
    Stat runtime = new Stat("runtime.ns");
    runtime.start();
    Source source = this.getSource();
    DeserializerProcessor deser = source.getDeserProcessor();
    List<OperationProcessor> operations = source.getOperationProcessors();
    List<String> containsStrings = source.getContainsStrings();
    List<Pattern> regexPatterns = source.getRegexPatterns();
    AtomicInteger totalEventsBytes = new AtomicInteger(0);
    AtomicInteger totalSerializedBytes = new AtomicInteger(0);
    this.getIpcService().setContext(context);
    Iterator<InternalEvent> events = this.getInternalEventIterator();
    /*
     * For logging purposes log when the function started running
     */
    this.monitor.invokeTimeNow();
    AtomicLong eventCount = new AtomicLong(0);
    AtomicLong oldestArrivalTime = new AtomicLong(System.currentTimeMillis());
    AtomicLong oldestOccurrenceTime = new AtomicLong(System.currentTimeMillis());
    /*
     * eventQueue allows for InternalEvents to be pulled from the Iterator and published to a
     * stream. A Thread is created that loops through events in the iterator and offers them to the
     * queue. Note that offering will be blocked if the queue is full (back pressure being applied).
     * When the iterator reaches the end (hasNext = false) the queue is closed.
     */
    this.eventQueue = new Queue<InternalEvent>(new LinkedBlockingQueue<InternalEvent>(this.queueSize));
    /*
     * Thread will live for duration of invocation and supply Stream with events.
     */
    new Thread(new Runnable() {

        @Override
        public void run() {
            while (events.hasNext()) {
                try {
                    eventQueue.offer(events.next());
                } catch (Queue.ClosedQueueException e) {
                    break;
                }
            }
            try {
                eventQueue.close();
            } catch (Queue.ClosedQueueException e) {
            }
        }
    }).start();
    Stream<InternalEvent> input = this.eventQueue.jdkStream();
    /*
     * Filter out raw events
     */
    Stream<InternalEvent> filtered = input.filter(/*
         * Perform regex filter
         */
    ievent -> {
        eventCount.incrementAndGet();
        String eventStr = ievent.getEventString();
        totalEventsBytes.addAndGet(eventStr.length());
        /*
           * Apply String contains filters before deserialization
           */
        for (String containsString : containsStrings) {
            if (eventStr.contains(containsString)) {
                return false;
            }
        }
        /*
           * Apply regex patterns before deserialization
           */
        for (Pattern regexPattern : regexPatterns) {
            Matcher m = regexPattern.matcher(eventStr);
            if (m.find()) {
                return false;
            }
        }
        return true;
    });
    /*
     * Deserialize
     */
    Stream<InternalEvent> deserialized = filtered.map(ievent -> {
        DeserializedEvent data = deser.deserialize(ievent.getEventString());
        if (data == null || data.getPayload() == null) {
            logger.warn("Failed to deserialize: " + ievent.getEventString());
            return null;
        }
        ievent.setEventObj(data);
        return ievent;
    }).filter(Objects::nonNull);
    /*
     * Perform Operations
     */
    Stream<InternalEvent> operated = deserialized;
    for (OperationProcessor operation : operations) {
        operated = operation.perform(operated);
    }
    /*
     * Serialize
     */
    Stream<InternalEvent> serialized = operated.map(ievent -> {
        try {
            String raw = this.ser.serialize(this.wrapper.getWrapped(ievent));
            totalSerializedBytes.addAndGet(raw.length());
            ievent.setSerialized(raw);
            return ievent;
        } catch (SerializationException e) {
            return null;
        }
    }).filter(Objects::nonNull);
    /*
     * Transport
     */
    AtomicBoolean abortEarly = new AtomicBoolean(false);
    serialized.takeWhile(internalEvent -> !abortEarly.get()).forEach(internalEvent -> {
        updateOldest(oldestArrivalTime, internalEvent.getArrivalTime());
        updateOldest(oldestOccurrenceTime, internalEvent.getEventTime());
        try {
            this.getIpcService().add(internalEvent);
        } catch (TransportException e) {
            logger.warn("error adding event", e);
            abortEarly.set(true);
        }
    });
    /*
     * Wait for transporters to finish
     */
    try {
        this.getIpcService().flush();
    } catch (TransportException e) {
        throw new HandlerException("encounted TransportException while shutting down ipcService", e);
    } catch (InterruptedException e) {
        throw new HandlerException("thread was interruptedwhile shutting down ipcService", e);
    } finally {
        String evtSource = this.getSourceName();
        runtime.stop();
        if (!this.skipWriteStats) {
            writeStats(eventCount.get(), oldestArrivalTime.get(), oldestOccurrenceTime.get(), evtSource, runtime, totalEventsBytes.get(), totalSerializedBytes.get());
        }
        if (logger.isTraceEnabled()) {
            getGCStats();
        }
    }
}