Examples with CollectorTestSink org.apache.apex.malhar.lib.testbench.CollectorTestSink used on opensource projects

Example 31 with CollectorTestSink

use of org.apache.apex.malhar.lib.testbench.CollectorTestSink in project apex-malhar by apache.

the class TopNTest method testNodeProcessingSchema.

@SuppressWarnings({ "rawtypes", "unchecked" })
public void testNodeProcessingSchema(TopN oper) {
    CollectorTestSink sortSink = new CollectorTestSink();
    oper.top.setSink(sortSink);
    oper.setN(3);
    oper.beginWindow(0);
    HashMap<String, Number> input = new HashMap<String, Number>();
    input.put("a", 2);
    oper.data.process(input);
    input.clear();
    input.put("a", 20);
    oper.data.process(input);
    input.clear();
    input.put("a", 1000);
    oper.data.process(input);
    input.clear();
    input.put("a", 5);
    oper.data.process(input);
    input.clear();
    input.put("a", 20);
    input.put("b", 33);
    oper.data.process(input);
    input.clear();
    input.put("a", 33);
    input.put("b", 34);
    oper.data.process(input);
    input.clear();
    input.put("b", 34);
    input.put("a", 1);
    oper.data.process(input);
    input.clear();
    input.put("b", 6);
    input.put("a", 1001);
    oper.data.process(input);
    input.clear();
    input.put("c", 9);
    oper.data.process(input);
    oper.endWindow();
    Assert.assertEquals("number emitted tuples", 3, sortSink.collectedTuples.size());
    for (Object o : sortSink.collectedTuples) {
        for (Map.Entry<String, ArrayList<Number>> e : ((HashMap<String, ArrayList<Number>>) o).entrySet()) {
            if (e.getKey().equals("a")) {
                Assert.assertEquals("emitted value for 'a' was ", 3, e.getValue().size());
            } else if (e.getKey().equals("b")) {
                Assert.assertEquals("emitted tuple for 'b' was ", 3, e.getValue().size());
            } else if (e.getKey().equals("c")) {
                Assert.assertEquals("emitted tuple for 'c' was ", 1, e.getValue().size());
            }
            log.debug(String.format("Sorted list for %s:", e.getKey()));
            for (Number i : e.getValue()) {
                log.debug(String.format("%s", i.toString()));
            }
        }
    }
    log.debug("Done testing round\n");
}

Example 32 with CollectorTestSink

use of org.apache.apex.malhar.lib.testbench.CollectorTestSink in project apex-malhar by apache.

the class QueryManagerAsynchronousTest method stressTest.

@Test
public void stressTest() throws Exception {
    final int totalTuples = 100000;
    final int batchSize = 100;
    final double waitMillisProb = .01;
    AppDataWindowEndQueueManager<MockQuery, Void> queueManager = new AppDataWindowEndQueueManager<MockQuery, Void>();
    DefaultOutputPort<String> outputPort = new DefaultOutputPort<String>();
    CollectorTestSink<MockResult> sink = new CollectorTestSink<MockResult>();
    TestUtils.setSink(outputPort, sink);
    MessageSerializerFactory msf = new MessageSerializerFactory(new ResultFormatter());
    QueryManagerAsynchronous<MockQuery, Void, MutableLong, MockResult> queryManagerAsynch = new QueryManagerAsynchronous<>(outputPort, queueManager, new NOPQueryExecutor(waitMillisProb), msf, Thread.currentThread());
    Thread producerThread = new Thread(new ProducerThread(queueManager, totalTuples, batchSize, waitMillisProb));
    producerThread.start();
    producerThread.setName("Producer Thread");
    long startTime = System.currentTimeMillis();
    queryManagerAsynch.setup(null);
    int numWindows = 0;
    for (; sink.collectedTuples.size() < totalTuples && ((System.currentTimeMillis() - startTime) < 60000); numWindows++) {
        queryManagerAsynch.beginWindow(numWindows);
        Thread.sleep(100);
        queryManagerAsynch.endWindow();
    }
    producerThread.stop();
    queryManagerAsynch.teardown();
    try {
        Thread.sleep(1000);
    } catch (InterruptedException e) {
    // Do Nothing
    }
    Assert.assertEquals(totalTuples, sink.collectedTuples.size());
}

Example 33 with CollectorTestSink

use of org.apache.apex.malhar.lib.testbench.CollectorTestSink in project apex-malhar by apache.

the class ProjectionTest method testProjection.

@Test
public void testProjection() {
    logger.debug("start round 0");
    projection.beginWindow(0);
    projection.input.process(data);
    Assert.assertEquals("projected tuples", 1, projection.projectedTuples);
    Assert.assertEquals("remainder tuples", 0, projection.remainderTuples);
    projection.endWindow();
    logger.debug("end round 0");
    CollectorTestSink projectedSink = new CollectorTestSink();
    CollectorTestSink remainderSink = new CollectorTestSink();
    projection.projected.setSink(projectedSink);
    projection.remainder.setSink(remainderSink);
    /* Collector Sink Test when remainder port is connected */
    logger.debug("start round 1");
    projection.beginWindow(1);
    data.setProjected(4321);
    data.setRemainder(9876);
    projection.input.process(data);
    Assert.assertEquals("projected tuples", 1, projection.projectedTuples);
    Assert.assertEquals("remainder tuples", 1, projection.remainderTuples);
    Object p = projectedSink.collectedTuples.get(0);
    Object r = remainderSink.collectedTuples.get(0);
    checkProjected(p, 4321);
    checkRemainder(r, 9876);
    projection.endWindow();
    logger.debug("end round 1");
}

Example 34 with CollectorTestSink

use of org.apache.apex.malhar.lib.testbench.CollectorTestSink in project apex-malhar by apache.

the class ApacheLogParseOperatorTest method testNodeProcessing.

/**
 * Test oper logic emits correct results
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testNodeProcessing() {
    ApacheLogParseOperator oper = new ApacheLogParseOperator();
    CollectorTestSink ipSink = new CollectorTestSink();
    CollectorTestSink urlSink = new CollectorTestSink();
    CollectorTestSink scSink = new CollectorTestSink();
    CollectorTestSink bytesSink = new CollectorTestSink();
    CollectorTestSink refSink = new CollectorTestSink();
    CollectorTestSink agentSink = new CollectorTestSink();
    oper.outputIPAddress.setSink(ipSink);
    oper.outputUrl.setSink(urlSink);
    oper.outputStatusCode.setSink(scSink);
    oper.outputBytes.setSink(bytesSink);
    oper.outputReferer.setSink(refSink);
    oper.outputAgent.setSink(agentSink);
    String token = "127.0.0.1 - - [04/Apr/2013:17:17:21 -0700] \"GET /favicon.ico HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.43 Safari/537.31\"";
    oper.beginWindow(0);
    oper.data.process(token);
    // 
    oper.endWindow();
    Assert.assertEquals("number emitted tuples", 1, ipSink.collectedTuples.size());
    Assert.assertEquals("number emitted tuples", 1, urlSink.collectedTuples.size());
    Assert.assertEquals("number emitted tuples", 1, scSink.collectedTuples.size());
    Assert.assertEquals("number emitted tuples", 1, bytesSink.collectedTuples.size());
    Assert.assertEquals("number emitted tuples", 1, refSink.collectedTuples.size());
    Assert.assertEquals("number emitted tuples", 1, agentSink.collectedTuples.size());
    log.debug(String.format("\nLine is \"%s\"", token));
    log.debug(String.format("IP is %s\n", ipSink.collectedTuples.toString()));
    log.debug(String.format("Url is %s\n", urlSink.collectedTuples.toString()));
    log.debug(String.format("Status code is %s\n", scSink.collectedTuples.toString()));
    log.debug(String.format("Bytes are %s\n", bytesSink.collectedTuples.toString()));
    log.debug(String.format("Referer is %s\n", refSink.collectedTuples.toString()));
    log.debug(String.format("Agent is %s\n", agentSink.collectedTuples.toString()));
}

Example 35 with CollectorTestSink

use of org.apache.apex.malhar.lib.testbench.CollectorTestSink in project apex-malhar by apache.

the class AverageTest method testNodeSchemaProcessing.

@SuppressWarnings({ "unchecked", "rawtypes" })
public void testNodeSchemaProcessing(Average oper) {
    CollectorTestSink averageSink = new CollectorTestSink();
    oper.average.setSink(averageSink);
    // 
    oper.beginWindow(0);
    Double a = new Double(2.0);
    Double b = new Double(20.0);
    Double c = new Double(1000.0);
    oper.data.process(a);
    oper.data.process(b);
    oper.data.process(c);
    a = 1.0;
    oper.data.process(a);
    a = 10.0;
    oper.data.process(a);
    b = 5.0;
    oper.data.process(b);
    b = 12.0;
    oper.data.process(b);
    c = 22.0;
    oper.data.process(c);
    c = 14.0;
    oper.data.process(c);
    a = 46.0;
    oper.data.process(a);
    b = 2.0;
    oper.data.process(b);
    a = 23.0;
    oper.data.process(a);
    // 
    oper.endWindow();
    Assert.assertEquals("number emitted tuples", 1, averageSink.collectedTuples.size());
    for (Object o : averageSink.collectedTuples) {
        // count is 12
        Number val = ((Pair<? extends Number, Integer>) o).getFirst().intValue();
        Assert.assertEquals("emitted average value was was ", new Integer(1157 / 12), val);
    }
}