Examples with CollectorTestSink org.apache.apex.malhar.lib.testbench.CollectorTestSink used on opensource projects

Example 81 with CollectorTestSink

use of org.apache.apex.malhar.lib.testbench.CollectorTestSink in project apex-malhar by apache.

the class DeduperTimeBasedPOJOImplTest method testDedupDifferentWindowSameKey.

@Test
public void testDedupDifferentWindowSameKey() {
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributes = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    attributes.put(DAG.APPLICATION_ID, APP_ID);
    attributes.put(DAG.APPLICATION_PATH, applicationPath);
    attributes.put(DAG.InputPortMeta.TUPLE_CLASS, TestPojo.class);
    OperatorContext context = mockOperatorContext(OPERATOR_ID, attributes);
    deduper.setup(context);
    deduper.input.setup(new PortContext(attributes, context));
    deduper.activate(context);
    CollectorTestSink<TestPojo> uniqueSink = new CollectorTestSink<TestPojo>();
    TestUtils.setSink(deduper.unique, uniqueSink);
    CollectorTestSink<TestPojo> duplicateSink = new CollectorTestSink<TestPojo>();
    TestUtils.setSink(deduper.duplicate, duplicateSink);
    CollectorTestSink<TestPojo> expiredSink = new CollectorTestSink<TestPojo>();
    TestUtils.setSink(deduper.expired, expiredSink);
    deduper.beginWindow(0);
    long millis = System.currentTimeMillis();
    deduper.input.process(new TestPojo(10, new Date(millis)));
    deduper.input.process(new TestPojo(11, new Date(millis + 10000)));
    deduper.input.process(new TestPojo(12, new Date(millis + 20000)));
    deduper.input.process(new TestPojo(13, new Date(millis + 30000)));
    deduper.input.process(new TestPojo(14, new Date(millis + 40000)));
    deduper.input.process(new TestPojo(15, new Date(millis + 50000)));
    // Duplicate
    deduper.input.process(new TestPojo(10, new Date(millis)));
    deduper.input.process(new TestPojo(16, new Date(millis + 60000)));
    // New tuple with same key but outside expired window.
    deduper.input.process(new TestPojo(10, new Date(millis + 70000)));
    // Earlier tuple with earlier time -- Expired
    deduper.input.process(new TestPojo(10, new Date(millis)));
    // New tuple repeated again - Duplicate
    deduper.input.process(new TestPojo(10, new Date(millis + 70000)));
    deduper.handleIdleTime();
    deduper.endWindow();
    Assert.assertTrue(uniqueSink.collectedTuples.size() == 8);
    Assert.assertTrue(duplicateSink.collectedTuples.size() == 2);
    Assert.assertTrue(expiredSink.collectedTuples.size() == 1);
    deduper.teardown();
}

Example 82 with CollectorTestSink

use of org.apache.apex.malhar.lib.testbench.CollectorTestSink in project apex-malhar by apache.

the class ApacheRandomLogsTest method test.

@SuppressWarnings({ "rawtypes", "unchecked", "deprecation" })
@Test
public void test() {
    ApacheGenRandomLogs oper = new ApacheGenRandomLogs();
    CollectorTestSink sink = new CollectorTestSink();
    oper.outport.setSink(sink);
    oper.setup(null);
    Thread t = new EmitTuples(oper);
    t.start();
    try {
        Thread.sleep(1000);
    } catch (InterruptedException e) {
    // Fixme
    }
    t.stop();
    Assert.assertTrue("Tuples emitted", sink.collectedTuples.size() > 0);
}

Example 83 with CollectorTestSink

use of org.apache.apex.malhar.lib.testbench.CollectorTestSink in project apex-malhar by apache.

the class HttpJsonChunksInputOperatorTest method testHttpInputModule.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testHttpInputModule() throws Exception {
    final List<String> receivedMessages = new ArrayList<String>();
    Handler handler = new AbstractHandler() {

        int responseCount = 0;

        @Override
        public void handle(String string, Request rq, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            IOUtils.copy(request.getInputStream(), bos);
            receivedMessages.add(new String(bos.toByteArray()));
            response.setContentType("application/json");
            response.setStatus(HttpServletResponse.SC_OK);
            response.setHeader("Transfer-Encoding", "chunked");
            try {
                JSONObject json = new JSONObject();
                json.put("responseId", "response" + ++responseCount);
                byte[] bytes = json.toString().getBytes();
                response.getOutputStream().println(bytes.length);
                response.getOutputStream().write(bytes);
                response.getOutputStream().println();
                response.getOutputStream().println(0);
                response.getOutputStream().flush();
            } catch (JSONException e) {
                response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Error generating response: " + e.toString());
            }
            ((Request) request).setHandled(true);
        }
    };
    Server server = new Server(0);
    server.setHandler(handler);
    server.start();
    String url = "http://localhost:" + server.getConnectors()[0].getLocalPort() + "/somecontext";
    final AbstractHttpInputOperator operator = new HttpJsonChunksInputOperator();
    CollectorTestSink sink = new CollectorTestSink();
    operator.outputPort.setSink(sink);
    operator.setUrl(new URI(url));
    operator.setup(null);
    operator.activate(null);
    int timeoutMillis = 3000;
    while (sink.collectedTuples.isEmpty() && timeoutMillis > 0) {
        operator.emitTuples();
        timeoutMillis -= 20;
        Thread.sleep(20);
    }
    Assert.assertTrue("tuple emitted", sink.collectedTuples.size() > 0);
    Map<String, String> tuple = (Map<String, String>) sink.collectedTuples.get(0);
    Assert.assertEquals("", tuple.get("responseId"), "response1");
    operator.deactivate();
    operator.teardown();
    server.stop();
}

Example 84 with CollectorTestSink

use of org.apache.apex.malhar.lib.testbench.CollectorTestSink in project apex-malhar by apache.

the class AbstractFileInputOperatorTest method testPartitioningStateTransferInterrupted.

/**
 * Test for testing dynamic partitioning.
 * - Create 4 file with 3 records each.
 * - Create a single partition, and read some records, populating pending files in operator.
 * - Split it in two operators
 * - Try to emit the remaining records.
 */
@Test
public void testPartitioningStateTransferInterrupted() throws Exception {
    LineByLineFileInputOperator oper = new LineByLineFileInputOperator();
    oper.getScanner().setFilePatternRegexp(".*partition([\\d]*)");
    oper.setDirectory(new File(testMeta.dir).getAbsolutePath());
    oper.setScanIntervalMillis(0);
    oper.setEmitBatchSize(2);
    LineByLineFileInputOperator initialState = new Kryo().copy(oper);
    // Create 4 files with 3 records each.
    Path path = new Path(new File(testMeta.dir).getAbsolutePath());
    FileContext.getLocalFSFileContext().delete(path, true);
    int file;
    for (file = 0; file < 4; file++) {
        FileUtils.write(new File(testMeta.dir, "partition00" + file), "a\nb\nc\n");
    }
    CollectorTestSink<String> queryResults = new CollectorTestSink<String>();
    @SuppressWarnings({ "unchecked", "rawtypes" }) CollectorTestSink<Object> sink = (CollectorTestSink) queryResults;
    oper.output.setSink(sink);
    int wid = 0;
    // Read some records
    oper.setup(testMeta.context);
    for (int i = 0; i < 5; i++) {
        oper.beginWindow(wid);
        oper.emitTuples();
        oper.endWindow();
        wid++;
    }
    Assert.assertEquals("Partial tuples read ", 6, sink.collectedTuples.size());
    Assert.assertEquals(1, initialState.getCurrentPartitions());
    initialState.setPartitionCount(2);
    StatsListener.Response rsp = initialState.processStats(null);
    Assert.assertEquals(true, rsp.repartitionRequired);
    // Create partitions of the operator.
    List<Partition<AbstractFileInputOperator<String>>> partitions = Lists.newArrayList();
    partitions.add(new DefaultPartition<AbstractFileInputOperator<String>>(oper));
    // incremental capacity controlled partitionCount property
    Collection<Partition<AbstractFileInputOperator<String>>> newPartitions = initialState.definePartitions(partitions, new PartitioningContextImpl(null, 0));
    Assert.assertEquals(2, newPartitions.size());
    Assert.assertEquals(1, initialState.getCurrentPartitions());
    Map<Integer, Partition<AbstractFileInputOperator<String>>> m = Maps.newHashMap();
    for (Partition<AbstractFileInputOperator<String>> p : newPartitions) {
        m.put(m.size(), p);
    }
    initialState.partitioned(m);
    Assert.assertEquals(2, initialState.getCurrentPartitions());
    /* Collect all operators in a list */
    List<AbstractFileInputOperator<String>> opers = Lists.newArrayList();
    for (Partition<AbstractFileInputOperator<String>> p : newPartitions) {
        LineByLineFileInputOperator oi = (LineByLineFileInputOperator) p.getPartitionedInstance();
        oi.setup(testMeta.context);
        oi.output.setSink(sink);
        opers.add(oi);
    }
    sink.clear();
    for (int i = 0; i < 10; i++) {
        for (AbstractFileInputOperator<String> o : opers) {
            o.beginWindow(wid);
            o.emitTuples();
            o.endWindow();
        }
        wid++;
    }
    Assert.assertEquals("Remaining tuples read ", 6, sink.collectedTuples.size());
}

Example 85 with CollectorTestSink

use of org.apache.apex.malhar.lib.testbench.CollectorTestSink in project apex-malhar by apache.

the class AbstractFileInputOperatorTest method testRecoveryWithFailedFile.

@Test
public void testRecoveryWithFailedFile() throws Exception {
    FileContext.getLocalFSFileContext().delete(new Path(new File(testMeta.dir).getAbsolutePath()), true);
    List<String> allLines = Lists.newArrayList();
    HashSet<String> lines = Sets.newHashSet();
    for (int line = 0; line < 5; line++) {
        lines.add("f0" + "l" + line);
    }
    allLines.addAll(lines);
    File testFile = new File(testMeta.dir, "file0");
    FileUtils.write(testFile, StringUtils.join(lines, '\n'));
    LineByLineFileInputOperator oper = new LineByLineFileInputOperator();
    oper.scanner = null;
    oper.failedFiles.add(new AbstractFileInputOperator.FailedFile(testFile.getAbsolutePath(), 1));
    CollectorTestSink<String> queryResults = new CollectorTestSink<String>();
    @SuppressWarnings({ "unchecked", "rawtypes" }) CollectorTestSink<Object> sink = (CollectorTestSink) queryResults;
    oper.output.setSink(sink);
    oper.setDirectory(testMeta.dir);
    oper.setup(testMeta.context);
    oper.beginWindow(0);
    oper.emitTuples();
    oper.endWindow();
    oper.teardown();
    Assert.assertEquals("number tuples", 4, queryResults.collectedTuples.size());
    Assert.assertEquals("lines", allLines.subList(1, allLines.size()), new ArrayList<String>(queryResults.collectedTuples));
}