Examples with Queue com.oath.cyclops.async.adapters.Queue used on opensource projects

Example 66 with Queue

use of com.oath.cyclops.async.adapters.Queue in project bender by Nextdoor.

the class ForkOperationTest method testEventCloning.

@Test
public void testEventCloning() {
    /*
     * Setup the pipeline of operation processors
     */
    List<List<OperationProcessor>> forks = new ArrayList<List<OperationProcessor>>();
    /*
     * Fork 1 that adds a "+"
     */
    List<OperationProcessor> fork1 = new ArrayList<OperationProcessor>();
    DummyAppendOperationFactory pos = new DummyAppendOperationFactory();
    DummyAppendOperationConfig posConf = new DummyAppendOperationConfig();
    posConf.setAppendStr("+");
    pos.setConf(posConf);
    fork1.add(new OperationProcessor(pos));
    forks.add(fork1);
    /*
     * Fork 2 that adds a "-"
     */
    List<OperationProcessor> fork2 = new ArrayList<OperationProcessor>();
    DummyAppendOperationFactory neg = new DummyAppendOperationFactory();
    DummyAppendOperationConfig negConf = new DummyAppendOperationConfig();
    negConf.setAppendStr("-");
    neg.setConf(negConf);
    fork2.add(new OperationProcessor(neg));
    forks.add(fork2);
    ForkOperation op = new ForkOperation(forks);
    /*
     * Create thread that supplies input events
     */
    Queue<InternalEvent> inputQueue = new Queue<InternalEvent>();
    supply(2, inputQueue);
    /*
     * Process
     */
    Stream<InternalEvent> input = inputQueue.stream();
    Stream<InternalEvent> output = op.getOutputStream(input);
    List<String> actual = output.map(m -> {
        return m.getEventObj().getPayload().toString();
    }).collect(Collectors.toList());
    List<String> expected = Arrays.asList("0+", "1+", "0-", "1-");
    assertEquals(4, actual.size());
    assertTrue(expected.containsAll(actual));
}

Example 67 with Queue

use of com.oath.cyclops.async.adapters.Queue in project bender by Nextdoor.

the class ForkOperationTest method testThreeForks.

@Test
public void testThreeForks() {
    /*
     * Setup the pipeline of operation processors
     */
    List<List<OperationProcessor>> forks = new ArrayList<List<OperationProcessor>>();
    List<OperationProcessor> fork1 = new ArrayList<OperationProcessor>();
    fork1.add(new OperationProcessor(new DummyThrottleOperationFactory()));
    forks.add(fork1);
    List<OperationProcessor> fork2 = new ArrayList<OperationProcessor>();
    fork2.add(new OperationProcessor(new DummyOperationFactory()));
    forks.add(fork2);
    List<OperationProcessor> fork3 = new ArrayList<OperationProcessor>();
    fork3.add(new OperationProcessor(new DummyOperationFactory()));
    forks.add(fork3);
    ForkOperation op = new ForkOperation(forks);
    /*
     * Create thread that supplies input events
     */
    Queue<InternalEvent> inputQueue = new Queue<InternalEvent>();
    supply(10, inputQueue);
    /*
     * Process
     */
    Stream<InternalEvent> input = inputQueue.stream();
    Stream<InternalEvent> output = op.getOutputStream(input);
    List<String> actual = output.map(InternalEvent::getEventString).collect(Collectors.toList());
    assertEquals(30, actual.size());
}

Example 68 with Queue

use of com.oath.cyclops.async.adapters.Queue in project bender by Nextdoor.

the class ForkOperation method getOutputStream.

/*-
   * This operation takes in an input Stream, copies each event from that Stream, and
   * writes each copy to a fork. Each fork has a consumer thread that pulls events
   * through the fork's Stream and outputs to a Queue. Finally a single thread consumes
   * the output Queue and writes to the output Stream. Visually this is what happens:
   * 
   *                       +--------------+
   *                       | Input Stream |
   *                       +-------+------+
   *                               |
   *                               v
   *                   +-----------+-----------+
   *                   | Input Consumer Thread |
   *                   +-----------+-----------+
   *                               |
   *                               v
   *                   +-----------+-----------+
   *                   |                       |
   *                   v                       v
   *            +------+-------+        +------+-------+
   *            | Fork 1 Queue |        | Fork 2 Queue |
   *            +------+-------+        +------+-------+
   *                   |                       |
   *                   v                       v
   *           +-------+-------+        +------+--------+
   *           | Fork 1 Stream |        | Fork 2 Stream |
   *           +-------+-------+        +------+--------+
   *                   |                       |
   *                   v                       v
   *       +-----------+-------+        +------+------------+
   *       | Fork 1 Operations |        | Fork 2 Operations |
   *       +-----------+-------+        +------+------------+
   *                   |                       |
   *                   v                       v
   *  +----------------+-------+        +------+-----------------+
   *  | Fork 1 Consumer Thread |        | Fork 2 Consumer Thread |
   *  +----------------+-------+        +------+-----------------+
   *                   |                       |
   *                   +-----------+-----------+
   *                               |
   *                               v
   *                       +-------+------+
   *                       | Output Queue |
   *                       +-------+------+
   *                               |
   *                               v
   *                  +------------+-----------+
   *                  | Output Consumer Thread |
   *                  +------------+-----------+
   *                               |
   *                               v
   *                       +-------+-------+
   *                       | Output Stream |
   *                       +---------------+
   * 
   */
public Stream<InternalEvent> getOutputStream(Stream<InternalEvent> input) {
    /*
     * forkOutputStreams keeps track of the output Stream of each fork.
     */
    List<Stream<InternalEvent>> forkOutputStreams = new ArrayList<Stream<InternalEvent>>(opProcsInForks.size());
    /*
     * From a list of operation configurations in each fork construct queues and streams.
     */
    this.queues = new ArrayList<Queue<InternalEvent>>(opProcsInForks.size());
    for (List<OperationProcessor> opProcsInFork : opProcsInForks) {
        /*
       * Construct a Queue for each fork. This is the input to each Fork.
       */
        Queue<InternalEvent> queue = new Queue<InternalEvent>(new LinkedBlockingQueue<InternalEvent>(opProcsInFork.size()));
        this.queues.add(queue);
        /*
       * Connect the fork's input Queue with operations. Each operation returns a stream with its
       * operation concatenated on.
       */
        Stream<InternalEvent> forkInput = queue.jdkStream();
        for (OperationProcessor opProcInFork : opProcsInFork) {
            forkInput = opProcInFork.perform(forkInput);
        }
        /*
       * Last input is the output.
       */
        forkOutputStreams.add(forkInput);
    }
    /*
     * Fork Consumer Threads
     * 
     * Combine each fork's output stream and write to the output Queue. When all data is consumed
     * the last fork closes the output Queue.
     */
    Queue<InternalEvent> outputQueue = new Queue<InternalEvent>(new LinkedBlockingQueue<InternalEvent>(this.queues.size()));
    AtomicInteger lock = new AtomicInteger(forkOutputStreams.size());
    forkOutputStreams.forEach(stream -> {
        this.es.execute(new StreamToQueue(stream, outputQueue, lock));
    });
    /*
     * Consume input Stream in a thread and publish to each fork's Queue.
     */
    new Thread(new Runnable() {

        @Override
        public void run() {
            input.forEach(ievent -> {
                queues.forEach(queue -> {
                    /*
             * The original event is NOT sent to each fork. Rather a copy of the event is sent to
             * each fork. This ensures that there is no contention between the operations performed
             * on each event. Caveat is that when the forks join there will be two events produced.
             */
                    queue.offer(ievent.copy());
                });
            });
            for (Queue<InternalEvent> queue : queues) {
                queue.close();
            }
        }
    }).start();
    return outputQueue.jdkStream();
}