Examples with FilterOperation com.nextdoor.bender.operation.FilterOperation used on opensource projects

Example 1 with FilterOperation

use of com.nextdoor.bender.operation.FilterOperation in project bender by Nextdoor.

the class ConditionalOperationTest method testTwoConditionsNoMatch.

@Test
public void testTwoConditionsNoMatch() {
    List<Pair<FilterOperation, List<OperationProcessor>>> conditions = new ArrayList<Pair<FilterOperation, List<OperationProcessor>>>();
    /*
     * Case 1
     */
    List<OperationProcessor> case1Ops = new ArrayList<OperationProcessor>();
    DummyAppendOperationFactory pos = new DummyAppendOperationFactory();
    DummyAppendOperationConfig posConf = new DummyAppendOperationConfig();
    posConf.setAppendStr("+");
    pos.setConf(posConf);
    case1Ops.add(new OperationProcessor(pos));
    FilterOperation case1Filter = new BasicFilterOperation(false);
    conditions.add(new ImmutablePair<FilterOperation, List<OperationProcessor>>(case1Filter, case1Ops));
    /*
     * Case 2
     */
    List<OperationProcessor> case2Ops = new ArrayList<OperationProcessor>();
    DummyAppendOperationFactory neg = new DummyAppendOperationFactory();
    DummyAppendOperationConfig negConf = new DummyAppendOperationConfig();
    negConf.setAppendStr("-");
    neg.setConf(negConf);
    case2Ops.add(new OperationProcessor(neg));
    FilterOperation case2Filter = new BasicFilterOperation(false);
    conditions.add(new ImmutablePair<FilterOperation, List<OperationProcessor>>(case2Filter, case2Ops));
    ConditionalOperation op = new ConditionalOperation(conditions, false);
    /*
     * Create thread that supplies input events
     */
    Queue<InternalEvent> inputQueue = new Queue<InternalEvent>();
    supply(2, inputQueue);
    /*
     * Process
     */
    Stream<InternalEvent> input = inputQueue.stream();
    Stream<InternalEvent> output = op.getOutputStream(input);
    List<String> actual = output.map(m -> {
        return m.getEventObj().getPayload().toString();
    }).collect(Collectors.toList());
    List<String> expected = Arrays.asList("0", "1");
    assertEquals(2, actual.size());
    assertTrue(expected.containsAll(actual));
}

Example 2 with FilterOperation

use of com.nextdoor.bender.operation.FilterOperation in project bender by Nextdoor.

the class ConditionalOperationTest method testFilterNonMatch.

public void testFilterNonMatch() {
    List<Pair<FilterOperation, List<OperationProcessor>>> conditions = new ArrayList<Pair<FilterOperation, List<OperationProcessor>>>();
    /*
     * Case 1
     */
    List<OperationProcessor> case1Ops = new ArrayList<OperationProcessor>();
    FilterOperation case1Filter = new BasicFilterOperation(false);
    conditions.add(new ImmutablePair<FilterOperation, List<OperationProcessor>>(case1Filter, case1Ops));
    ConditionalOperation op = new ConditionalOperation(conditions, true);
    /*
     * Create thread that supplies input events
     */
    Queue<InternalEvent> inputQueue = new Queue<InternalEvent>();
    supply(2, inputQueue);
    /*
     * Process
     */
    Stream<InternalEvent> input = inputQueue.stream();
    Stream<InternalEvent> output = op.getOutputStream(input);
    List<String> actual = output.map(m -> {
        return m.getEventObj().getPayload().toString();
    }).collect(Collectors.toList());
    assertEquals(0, actual.size());
}

Example 3 with FilterOperation

use of com.nextdoor.bender.operation.FilterOperation in project bender by Nextdoor.

the class ConditionalOperationTest method testFilterCondition.

@Test
public void testFilterCondition() {
    List<Pair<FilterOperation, List<OperationProcessor>>> conditions = new ArrayList<Pair<FilterOperation, List<OperationProcessor>>>();
    /*
     * Case 1
     */
    List<OperationProcessor> case1Ops = new ArrayList<OperationProcessor>();
    BasicFilterOperationFactory fOp = new BasicFilterOperationFactory();
    BasicFilterOperationConfig fOpConf = new BasicFilterOperationConfig();
    fOpConf.setPass(false);
    fOp.setConf(fOpConf);
    case1Ops.add(new OperationProcessor(fOp));
    FilterOperation case1Filter = new BasicFilterOperation(true);
    conditions.add(new ImmutablePair<FilterOperation, List<OperationProcessor>>(case1Filter, case1Ops));
    ConditionalOperation op = new ConditionalOperation(conditions, false);
    /*
     * Create thread that supplies input events
     */
    Queue<InternalEvent> inputQueue = new Queue<InternalEvent>();
    supply(2, inputQueue);
    /*
     * Process
     */
    Stream<InternalEvent> input = inputQueue.stream();
    Stream<InternalEvent> output = op.getOutputStream(input);
    List<String> actual = output.map(m -> {
        return m.getEventObj().getPayload().toString();
    }).collect(Collectors.toList());
    assertEquals(0, actual.size());
}

Example 4 with FilterOperation

use of com.nextdoor.bender.operation.FilterOperation in project bender by Nextdoor.

the class ConditionalOperation method getOutputStream.

/*-
   * This operation takes in an input Stream of events and checks the event against
   * each condition in an if elseif manner. The first matching condition is send the 
   * event. If no conditions match and filter non-match is specified true then the
   * event is filtered out. If false the event is sent to the output queue. 
   * 
   *           +--------------+
   *           | Input Stream |
   *           +------+-------+
   *                  |
   *                  v
   *       +----------+-----------+
   *       | Input Consumer Thread |
   *       +----------+------------+
   *                  |
   *        +---------+
   *        |
   *        v
   *  +-----+-------+    +-------------+      +-------------+
   *  | Condition 1 | No | Condition 2 |  No  |    Filter   |
   *  |   Filter    +--->+   Filter    +----->+  Non-Match  |
   *  +-----+-------+    +-----+-------+      +------+------+
   *        |                  |                     |
   *    Yes |             Yes  |                     | No
   *        v                  v                     |
   *   +----+-----+       +----+-----+               |
   *   |  Queue 1 |       |  Queue 2 |               |
   *   +----+-----+       +----+-----+               |
   *        |                  |                     |
   *        v                  v                     |
   *    +---+----+         +---+----+                |
   *    | Stream |         | Stream |                |
   *    +---+----+         +---+----+                |
   *        |                  |                     |
   *        v                  v                     |
   *  +-----+------+     +-----+------+              |
   *  | Operations |     | Operations |              |
   *  +-----+------+     +-----+------+              |
   *        |                  |                     |
   *        v                  v                     |
   *   +----+-----+       +----+-----+               |
   *   | Consumer |       | Consumer |               |
   *   |  Thread  |       |  Thread  |               |
   *   +--+-------+       +-----+----+               |
   *      |                     |                    |
   *      |   +--------------+  |                    |
   *      +-->+ Output Queue +<-+--------------------+
   *          +------+-------+
   *                 |
   *                 v
   *     +-----------+------------+
   *     | Output Consumer Thread |
   *     +-----------+------------+
   *                 |
   *                 v
   *         +-------+-------+
   *         | Output Stream |
   *         +---------------+
   *  
   * 
   */
public Stream<InternalEvent> getOutputStream(Stream<InternalEvent> input) {
    /*
     * outputStreams keeps track of the output Stream of each Condition.
     */
    List<Stream<InternalEvent>> outputStreams = new ArrayList<Stream<InternalEvent>>(this.conditionsAndProcs.size());
    /*
     * From a list of operation configurations in each condition construct queues and streams.
     */
    this.filtersAndQueues = new ArrayList<Pair<FilterOperation, Queue<InternalEvent>>>(this.conditionsAndProcs.size());
    for (Pair<FilterOperation, List<OperationProcessor>> filterAndProcs : this.conditionsAndProcs) {
        FilterOperation filter = filterAndProcs.getLeft();
        List<OperationProcessor> procs = filterAndProcs.getRight();
        /*
       * Construct a Queue for each conditional. This is the input to each Condition.
       */
        Queue<InternalEvent> queue = new Queue<InternalEvent>(new LinkedBlockingQueue<InternalEvent>(procs.size()));
        this.filtersAndQueues.add(new ImmutablePair<FilterOperation, Queue<InternalEvent>>(filter, queue));
        /*
       * Connect the condition's input Queue with operations. Each operation returns a stream with its
       * operation concatenated on.
       */
        Stream<InternalEvent> conditionInput = queue.jdkStream();
        for (OperationProcessor proc : procs) {
            conditionInput = proc.perform(conditionInput);
        }
        /*
       * Last input is the output.
       */
        outputStreams.add(conditionInput);
    }
    /*
     * Condition Consumer Threads
     * 
     * Combine each condition's output stream and write to the output Queue. When all data is consumed
     * the last condition closes the output Queue.
     */
    Queue<InternalEvent> outputQueue = new Queue<InternalEvent>(new LinkedBlockingQueue<InternalEvent>(this.conditionsAndProcs.size()));
    AtomicInteger lock = new AtomicInteger(outputStreams.size());
    outputStreams.forEach(stream -> {
        this.es.execute(new StreamToQueue(stream, outputQueue, lock));
    });
    /*
     * Consume input Stream in a thread and publish to each condition's Queue.
     */
    new Thread(new Runnable() {

        @Override
        public void run() {
            input.forEach(ievent -> {
                boolean matches = false;
                for (Pair<FilterOperation, Queue<InternalEvent>> filterAndQueue : filtersAndQueues) {
                    FilterOperation filter = filterAndQueue.getLeft();
                    /*
             * If event passes the filter offer event to queue.
             */
                    if (filter.test(ievent)) {
                        filterAndQueue.getRight().offer(ievent);
                        matches = true;
                        break;
                    }
                }
                /*
           * Send to output queue if no case matches
           */
                if (!matches && !filterNonMatch) {
                    outputQueue.offer(ievent);
                }
            });
            /*
         * Close queues when source queue is consumed.
         */
            for (Pair<FilterOperation, Queue<InternalEvent>> filterAndQueue : filtersAndQueues) {
                filterAndQueue.getRight().close();
            }
        }
    }).start();
    return outputQueue.jdkStream();
}

Example 5 with FilterOperation

use of com.nextdoor.bender.operation.FilterOperation in project bender by Nextdoor.

the class ConditionalOperationFactory method setConf.

@Override
public void setConf(AbstractConfig config) {
    this.config = (ConditionalOperationConfig) config;
    List<Pair<FilterOperation, List<OperationProcessor>>> cases = new ArrayList<Pair<FilterOperation, List<OperationProcessor>>>();
    OperationFactoryFactory off = new OperationFactoryFactory();
    for (Condition caze : this.config.getConditions()) {
        List<OperationProcessor> processorsInCase = new ArrayList<OperationProcessor>();
        /*
       * Create {@OperationProcessor}s from configs
       */
        for (OperationConfig opConfig : caze.getOperations()) {
            try {
                processorsInCase.add(new OperationProcessor(off.getFactory(opConfig)));
            } catch (ClassNotFoundException e) {
                throw new RuntimeException(e);
            }
        }
        FilterOperation filter;
        try {
            filter = (FilterOperation) off.getFactory(caze.getCondition()).newInstance();
        } catch (ClassNotFoundException e) {
            throw new RuntimeException(e);
        }
        cases.add(new ImmutablePair<FilterOperation, List<OperationProcessor>>(filter, processorsInCase));
    }
    this.cases = cases;
}