Examples with FSWindowDataManager org.apache.apex.malhar.lib.wal.FSWindowDataManager used on opensource projects

Example 6 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class FileSplitterInputTest method testIdempotencyWithBlocksThreshold.

@Test
public void testIdempotencyWithBlocksThreshold() throws InterruptedException {
    FSWindowDataManager fsWindowDataManager = new FSWindowDataManager();
    testMeta.updateConfig(fsWindowDataManager, 500, 2L, 10);
    testMeta.fileSplitterInput.setup(testMeta.context);
    validateRecovery(8, 2);
    testMeta.fileSplitterInput.teardown();
}

Example 7 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class KafkaOutputOperatorTest method cleanUp.

private void cleanUp(Context.OperatorContext operatorContext) {
    FSWindowDataManager windowDataManager = new FSWindowDataManager();
    windowDataManager.setup(operatorContext);
    try {
        windowDataManager.committed(windowDataManager.getLargestCompletedWindow());
    } catch (IOException e) {
        e.printStackTrace();
    }
}

Example 8 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class KafkaInputOperatorTest method testInputOperator.

public void testInputOperator(boolean hasFailure, boolean idempotent) throws Exception {
    // each broker should get a END_TUPLE message
    latch = new CountDownLatch(countDownAll ? totalCount + totalBrokers : totalBrokers);
    logger.info("Test Case: name: {}; totalBrokers: {}; hasFailure: {}; hasMultiCluster: {};" + " hasMultiPartition: {}, partition: {}", testName, totalBrokers, hasFailure, hasMultiCluster, hasMultiPartition, partition);
    // Start producer
    KafkaTestProducer p = new KafkaTestProducer(testName, hasMultiPartition, hasMultiCluster);
    p.setSendCount(totalCount);
    Thread t = new Thread(p);
    t.start();
    int expectedReceiveCount = totalCount + totalBrokers;
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    // Create KafkaSinglePortStringInputOperator
    KafkaSinglePortInputOperator node = dag.addOperator("Kafka input" + testName, KafkaSinglePortInputOperator.class);
    node.setInitialPartitionCount(1);
    // set topic
    node.setTopics(testName);
    node.setInitialOffset(AbstractKafkaInputOperator.InitialOffset.EARLIEST.name());
    node.setClusters(getClusterConfig());
    node.setStrategy(partition);
    if (idempotent) {
        node.setWindowDataManager(new FSWindowDataManager());
    }
    // Create Test tuple collector
    CollectorModule collector = dag.addOperator("TestMessageCollector", CollectorModule.class);
    collector.isIdempotentTest = idempotent;
    // Connect ports
    dag.addStream("Kafka message" + testName, node.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    if (hasFailure) {
        setupHasFailureTest(node, dag);
    }
    // Create local cluster
    LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    // let the Controller to run the inside another thread. It is almost same as call Controller.runAsync(),
    // but Controller.runAsync() don't expose the thread which run it,
    // so we don't know when the thread will be terminated.
    // create this thread and then call join() to make sure the Controller shutdown completely.
    monitorThread = new Thread((StramLocalCluster) lc, "master");
    monitorThread.start();
    boolean notTimeout = true;
    try {
        // Wait 60s for consumer finish consuming all the messages
        notTimeout = latch.await(waitTime, TimeUnit.MILLISECONDS);
        lc.shutdown();
        // wait until control thread finished.
        monitorThread.join();
    } catch (Exception e) {
        logger.warn(e.getMessage());
    }
    t.join();
    if (!notTimeout || expectedReceiveCount != tupleCollection.size()) {
        logger.info("Number of received/expected tuples: {}/{}, testName: {}, tuples: \n{}", tupleCollection.size(), expectedReceiveCount, testName, tupleCollection);
    }
    Assert.assertTrue("TIMEOUT. testName: " + this.testName + "; Collected data: " + tupleCollection, notTimeout);
    // Check results
    Assert.assertTrue("testName: " + testName + "; Collected tuple size: " + tupleCollection.size() + "; Expected tuple size: " + expectedReceiveCount + "; data: \n" + tupleCollection, expectedReceiveCount == tupleCollection.size());
    logger.info("End of test case: {}", testName);
}

Example 9 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class KafkaInputOperatorTest method testKafkaInputOperator.

/**
 * Test AbstractKafkaSinglePortInputOperator (i.e. an input adapter for
 * Kafka, aka consumer). This module receives data from an outside test
 * generator through Kafka message bus and feed that data into Malhar
 * streaming platform.
 *
 * [Generate message and send that to Kafka message bus] ==> [Receive that
 * message through Kafka input adapter(i.e. consumer) and send using
 * emitTuples() interface on output port during onMessage call]
 *
 * @throws Exception
 */
public void testKafkaInputOperator(int sleepTime, final int totalCount, KafkaConsumer consumer, boolean isValid, boolean idempotent) throws Exception {
    // initial the latch for this test
    latch = new CountDownLatch(1);
    // Start producer
    KafkaTestProducer p = new KafkaTestProducer(TEST_TOPIC);
    p.setSendCount(totalCount);
    new Thread(p).start();
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    // Create KafkaSinglePortStringInputOperator
    KafkaSinglePortStringInputOperator node = dag.addOperator("Kafka message consumer", KafkaSinglePortStringInputOperator.class);
    if (isSuicide) {
        // make some extreme assumptions to make it fail if checkpointing wrong offsets
        dag.setAttribute(Context.DAGContext.CHECKPOINT_WINDOW_COUNT, 1);
        dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new FSStorageAgent("target/ck", new Configuration()));
        node.setMaxTuplesPerWindow(500);
    }
    if (idempotent) {
        node.setWindowDataManager(new FSWindowDataManager());
    }
    consumer.setTopic(TEST_TOPIC);
    node.setConsumer(consumer);
    consumer.setCacheSize(5000);
    if (isValid) {
        node.setZookeeper("localhost:" + KafkaOperatorTestBase.TEST_ZOOKEEPER_PORT[0]);
    }
    // Create Test tuple collector
    CollectorModule<String> collector = dag.addOperator("TestMessageCollector", new CollectorModule<String>());
    // Connect ports
    dag.addStream("Kafka message", node.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    lc.runAsync();
    // Wait 30s for consumer finish consuming all the messages
    Assert.assertTrue("TIMEOUT: 30s ", latch.await(300000, TimeUnit.MILLISECONDS));
    // Check results
    Assert.assertTrue("Expected count >= " + totalCount + "; Actual count " + tupleCount.intValue(), totalCount <= tupleCount.intValue());
    logger.debug(String.format("Number of emitted tuples: %d", tupleCount.intValue()));
    p.close();
    lc.shutdown();
}

Example 10 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class KafkaInputOperatorTest method createOperator.

private KafkaSinglePortStringInputOperator createOperator(boolean isIdempotency) {
    Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
    attributeMap.put(Context.OperatorContext.SPIN_MILLIS, 500);
    attributeMap.put(Context.DAGContext.APPLICATION_PATH, testMeta.baseDir);
    testMeta.context = mockOperatorContext(1, attributeMap);
    testMeta.operator = new KafkaSinglePortStringInputOperator();
    KafkaConsumer consumer = new SimpleKafkaConsumer();
    consumer.setTopic(TEST_TOPIC);
    consumer.setInitialOffset("earliest");
    if (isIdempotency) {
        FSWindowDataManager storageManager = new FSWindowDataManager();
        storageManager.setStatePath(testMeta.recoveryDir);
        testMeta.operator.setWindowDataManager(storageManager);
    }
    testMeta.operator.setConsumer(consumer);
    testMeta.operator.setZookeeper("localhost:" + KafkaOperatorTestBase.TEST_ZOOKEEPER_PORT[0]);
    testMeta.operator.setMaxTuplesPerWindow(500);
    List<Partitioner.Partition<AbstractKafkaInputOperator<KafkaConsumer>>> partitions = new LinkedList<Partitioner.Partition<AbstractKafkaInputOperator<KafkaConsumer>>>();
    Collection<Partitioner.Partition<AbstractKafkaInputOperator<KafkaConsumer>>> newPartitions = testMeta.operator.definePartitions(partitions, new StatelessPartitionerTest.PartitioningContextImpl(null, 0));
    Assert.assertEquals(1, newPartitions.size());
    KafkaSinglePortStringInputOperator operator = (KafkaSinglePortStringInputOperator) newPartitions.iterator().next().getPartitionedInstance();
    testMeta.sink = new CollectorTestSink<Object>();
    testMeta.operator.outputPort.setSink(testMeta.sink);
    operator.outputPort.setSink(testMeta.sink);
    return operator;
}