Examples with Queue com.oath.cyclops.async.adapters.Queue used on opensource projects

Example 56 with Queue

use of com.oath.cyclops.async.adapters.Queue in project bender by Nextdoor.

the class ConditionalOperation method getOutputStream.

/*-
   * This operation takes in an input Stream of events and checks the event against
   * each condition in an if elseif manner. The first matching condition is send the 
   * event. If no conditions match and filter non-match is specified true then the
   * event is filtered out. If false the event is sent to the output queue. 
   * 
   *           +--------------+
   *           | Input Stream |
   *           +------+-------+
   *                  |
   *                  v
   *       +----------+-----------+
   *       | Input Consumer Thread |
   *       +----------+------------+
   *                  |
   *        +---------+
   *        |
   *        v
   *  +-----+-------+    +-------------+      +-------------+
   *  | Condition 1 | No | Condition 2 |  No  |    Filter   |
   *  |   Filter    +--->+   Filter    +----->+  Non-Match  |
   *  +-----+-------+    +-----+-------+      +------+------+
   *        |                  |                     |
   *    Yes |             Yes  |                     | No
   *        v                  v                     |
   *   +----+-----+       +----+-----+               |
   *   |  Queue 1 |       |  Queue 2 |               |
   *   +----+-----+       +----+-----+               |
   *        |                  |                     |
   *        v                  v                     |
   *    +---+----+         +---+----+                |
   *    | Stream |         | Stream |                |
   *    +---+----+         +---+----+                |
   *        |                  |                     |
   *        v                  v                     |
   *  +-----+------+     +-----+------+              |
   *  | Operations |     | Operations |              |
   *  +-----+------+     +-----+------+              |
   *        |                  |                     |
   *        v                  v                     |
   *   +----+-----+       +----+-----+               |
   *   | Consumer |       | Consumer |               |
   *   |  Thread  |       |  Thread  |               |
   *   +--+-------+       +-----+----+               |
   *      |                     |                    |
   *      |   +--------------+  |                    |
   *      +-->+ Output Queue +<-+--------------------+
   *          +------+-------+
   *                 |
   *                 v
   *     +-----------+------------+
   *     | Output Consumer Thread |
   *     +-----------+------------+
   *                 |
   *                 v
   *         +-------+-------+
   *         | Output Stream |
   *         +---------------+
   *  
   * 
   */
public Stream<InternalEvent> getOutputStream(Stream<InternalEvent> input) {
    /*
     * outputStreams keeps track of the output Stream of each Condition.
     */
    List<Stream<InternalEvent>> outputStreams = new ArrayList<Stream<InternalEvent>>(this.conditionsAndProcs.size());
    /*
     * From a list of operation configurations in each condition construct queues and streams.
     */
    this.filtersAndQueues = new ArrayList<Pair<FilterOperation, Queue<InternalEvent>>>(this.conditionsAndProcs.size());
    for (Pair<FilterOperation, List<OperationProcessor>> filterAndProcs : this.conditionsAndProcs) {
        FilterOperation filter = filterAndProcs.getLeft();
        List<OperationProcessor> procs = filterAndProcs.getRight();
        /*
       * Construct a Queue for each conditional. This is the input to each Condition.
       */
        Queue<InternalEvent> queue = new Queue<InternalEvent>(new LinkedBlockingQueue<InternalEvent>(procs.size()));
        this.filtersAndQueues.add(new ImmutablePair<FilterOperation, Queue<InternalEvent>>(filter, queue));
        /*
       * Connect the condition's input Queue with operations. Each operation returns a stream with its
       * operation concatenated on.
       */
        Stream<InternalEvent> conditionInput = queue.jdkStream();
        for (OperationProcessor proc : procs) {
            conditionInput = proc.perform(conditionInput);
        }
        /*
       * Last input is the output.
       */
        outputStreams.add(conditionInput);
    }
    /*
     * Condition Consumer Threads
     * 
     * Combine each condition's output stream and write to the output Queue. When all data is consumed
     * the last condition closes the output Queue.
     */
    Queue<InternalEvent> outputQueue = new Queue<InternalEvent>(new LinkedBlockingQueue<InternalEvent>(this.conditionsAndProcs.size()));
    AtomicInteger lock = new AtomicInteger(outputStreams.size());
    outputStreams.forEach(stream -> {
        this.es.execute(new StreamToQueue(stream, outputQueue, lock));
    });
    /*
     * Consume input Stream in a thread and publish to each condition's Queue.
     */
    new Thread(new Runnable() {

        @Override
        public void run() {
            input.forEach(ievent -> {
                boolean matches = false;
                for (Pair<FilterOperation, Queue<InternalEvent>> filterAndQueue : filtersAndQueues) {
                    FilterOperation filter = filterAndQueue.getLeft();
                    /*
             * If event passes the filter offer event to queue.
             */
                    if (filter.test(ievent)) {
                        filterAndQueue.getRight().offer(ievent);
                        matches = true;
                        break;
                    }
                }
                /*
           * Send to output queue if no case matches
           */
                if (!matches && !filterNonMatch) {
                    outputQueue.offer(ievent);
                }
            });
            /*
         * Close queues when source queue is consumed.
         */
            for (Pair<FilterOperation, Queue<InternalEvent>> filterAndQueue : filtersAndQueues) {
                filterAndQueue.getRight().close();
            }
        }
    }).start();
    return outputQueue.jdkStream();
}

Example 57 with Queue

use of com.oath.cyclops.async.adapters.Queue in project bender by Nextdoor.

the class BaseHandlerQueueTest method testSlowSourceFastConsumer.

@Test
public void testSlowSourceFastConsumer() throws HandlerException {
    BaseHandler.CONFIG_FILE = "/config/handler_config_queue.yaml";
    Queue<DummyEvent> q = new Queue<DummyEvent>();
    Iterator<DummyEvent> dummyEvents = q.stream().iterator();
    new Thread(new Runnable() {

        @Override
        public void run() {
            for (int i = 0; i < 1000; i++) {
                if (i % 500 == 0) {
                    try {
                        Thread.sleep(1000);
                    } catch (InterruptedException e) {
                    }
                }
                q.offer(new DummyEvent("" + i, 0));
            }
            q.close();
        }
    }).start();
    TestContext context = new TestContext();
    context.setInvokedFunctionArn("arn:aws:lambda:us-east-1:123:function:test:tag");
    handler.handler(dummyEvents, context);
    /*
     * Verify Events made it all the way through
     */
    assertEquals(1000, BufferedTransporter.output.size());
}

Example 58 with Queue

use of com.oath.cyclops.async.adapters.Queue in project bender by Nextdoor.

the class BaseHandlerQueueTest method testFastSourceFastConsumer.

@Test
public void testFastSourceFastConsumer() throws HandlerException {
    BaseHandler.CONFIG_FILE = "/config/handler_config_queue.yaml";
    Queue<DummyEvent> q = new Queue<DummyEvent>();
    Iterator<DummyEvent> dummyEvents = q.stream().iterator();
    for (int i = 0; i < 1000; i++) {
        q.offer(new DummyEvent("" + i, 0));
    }
    q.close();
    TestContext context = new TestContext();
    context.setInvokedFunctionArn("arn:aws:lambda:us-east-1:123:function:test:tag");
    handler.handler(dummyEvents, context);
    /*
     * Verify Events made it all the way through
     */
    assertEquals(1000, BufferedTransporter.output.size());
}

Example 59 with Queue

use of com.oath.cyclops.async.adapters.Queue in project bender by Nextdoor.

the class BaseHandlerQueueTest method testFastSourceSlowConsumer.

@Test
public void testFastSourceSlowConsumer() throws HandlerException {
    BaseHandler.CONFIG_FILE = "/config/handler_config_queue_throttle.yaml";
    Queue<DummyEvent> q = new Queue<DummyEvent>();
    Iterator<DummyEvent> dummyEvents = q.stream().iterator();
    for (int i = 0; i < 1000; i++) {
        q.offer(new DummyEvent("" + i, 0));
    }
    q.close();
    TestContext context = new TestContext();
    context.setInvokedFunctionArn("arn:aws:lambda:us-east-1:123:function:test:tag");
    handler.handler(dummyEvents, context);
    /*
     * Verify Events made it all the way through
     */
    assertEquals(1000, BufferedTransporter.output.size());
}

Example 60 with Queue

use of com.oath.cyclops.async.adapters.Queue in project bender by Nextdoor.

the class ConditionalOperationTest method testSingleConditionMatch.

@Test
public void testSingleConditionMatch() {
    /*
     * Setup the pipeline of operation processors
     */
    List<Pair<FilterOperation, List<OperationProcessor>>> conditions = new ArrayList<Pair<FilterOperation, List<OperationProcessor>>>();
    List<OperationProcessor> case1Ops = new ArrayList<OperationProcessor>();
    DummyAppendOperationFactory pos = new DummyAppendOperationFactory();
    DummyAppendOperationConfig posConf = new DummyAppendOperationConfig();
    posConf.setAppendStr("+");
    pos.setConf(posConf);
    case1Ops.add(new OperationProcessor(pos));
    FilterOperation filter = new BasicFilterOperation(true);
    conditions.add(new ImmutablePair<FilterOperation, List<OperationProcessor>>(filter, case1Ops));
    ConditionalOperation op = new ConditionalOperation(conditions, false);
    /*
     * Create thread that supplies input events
     */
    Queue<InternalEvent> inputQueue = new Queue<InternalEvent>();
    supply(2, inputQueue);
    /*
     * Process
     */
    Stream<InternalEvent> input = inputQueue.stream();
    Stream<InternalEvent> output = op.getOutputStream(input);
    List<String> actual = output.map(m -> {
        return m.getEventObj().getPayload().toString();
    }).collect(Collectors.toList());
    List<String> expected = Arrays.asList("0+", "1+");
    assertEquals(2, actual.size());
    assertTrue(expected.containsAll(actual));
}