Examples with OperatorContext com.datatorrent.api.Context.OperatorContext used on opensource projects

Example 26 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class JdbcNonTransactionalOutputOperatorTest method testJdbcOutputOperator.

@Test
public void testJdbcOutputOperator() {
    JdbcStore store = new JdbcStore();
    store.setDatabaseDriver(DB_DRIVER);
    store.setDatabaseUrl(URL);
    TestOutputOperator outputOperator = new TestOutputOperator();
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributeMap = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    attributeMap.put(DAG.APPLICATION_ID, APP_ID);
    OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);
    outputOperator.setStore(store);
    outputOperator.setup(context);
    List<TestEvent> events = Lists.newArrayList();
    for (int i = 0; i < 10; i++) {
        events.add(new TestEvent(i));
    }
    outputOperator.beginWindow(0);
    for (TestEvent event : events) {
        outputOperator.input.process(event);
    }
    outputOperator.endWindow();
    Assert.assertEquals("rows in db", 10, outputOperator.getNumOfEventsInStore());
}

Example 27 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class JdbcNonTransactionalBatchOutputOperatorTest method testAtMostOnceHalfBatch.

@Test
public void testAtMostOnceHalfBatch() {
    JdbcOperatorTest.cleanTable();
    Random random = new Random();
    TestOutputOperator outputOperator = createOperator(ProcessingMode.AT_MOST_ONCE);
    outputOperator.beginWindow(0);
    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
        outputOperator.input.put(new TestEvent(random.nextInt()));
    }
    outputOperator.endWindow();
    Assert.assertEquals("Commit window id ", 0, outputOperator.getStore().getCommittedWindowId(APP_ID, OPERATOR_ID));
    Assert.assertEquals("Batch should be written", BATCH_SIZE, outputOperator.getNumOfEventsInStore(outputOperator.getStore().connection));
    outputOperator.beginWindow(1);
    for (int batchCounter = 0; batchCounter < HALF_BATCH_SIZE; batchCounter++) {
        outputOperator.input.put(new TestEvent(random.nextInt()));
    }
    Assert.assertEquals("Commit window id ", 0, outputOperator.getStore().getCommittedWindowId(APP_ID, OPERATOR_ID));
    Assert.assertEquals("Batch should be written", BATCH_SIZE, outputOperator.getNumOfEventsInStore(outputOperator.getStore().connection));
    outputOperator.getStore().disconnect();
    // //
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributeMap = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_MOST_ONCE);
    attributeMap.put(OperatorContext.ACTIVATION_WINDOW_ID, 0L);
    attributeMap.put(DAG.APPLICATION_ID, APP_ID);
    OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);
    outputOperator.setup(context);
    Assert.assertEquals("Commit window id ", 0, outputOperator.getStore().getCommittedWindowId(APP_ID, OPERATOR_ID));
    Assert.assertEquals("Batch should be written", BATCH_SIZE, outputOperator.getNumOfEventsInStore(outputOperator.getStore().connection));
    outputOperator.beginWindow(2);
    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
        outputOperator.input.put(new TestEvent(random.nextInt()));
    }
    outputOperator.endWindow();
    Assert.assertEquals("Commit window id ", 2, outputOperator.getStore().getCommittedWindowId(APP_ID, OPERATOR_ID));
    Assert.assertEquals("Batch should be written", 2 * BATCH_SIZE, outputOperator.getNumOfEventsInStore(outputOperator.getStore().connection));
}

Example 28 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class AerospikeOperatorTest method TestAerospikeOutputOperator.

@Test
public void TestAerospikeOutputOperator() {
    AerospikeTransactionalStore transactionalStore = getTransactionalStore();
    OperatorContext context = getOperatorContext(APP_ID);
    TestOutputOperator outputOperator = new TestOutputOperator();
    outputOperator.setStore(transactionalStore);
    outputOperator.setup(context);
    List<TestEvent> events = new ArrayList<TestEvent>();
    for (int i = 0; i < NUM_TUPLES; i++) {
        events.add(new TestEvent(i));
    }
    outputOperator.beginWindow(0);
    for (TestEvent event : events) {
        outputOperator.input.process(event);
    }
    outputOperator.endWindow();
    Assert.assertEquals("rows in db", NUM_TUPLES, getNumOfEventsInStore());
}

Example 29 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class KafkaOutputOperatorTest method sendDataToKafka.

private void sendDataToKafka(boolean exactlyOnce, List<Person> toKafka, boolean hasFailure, boolean differentTuplesAfterRecovery) throws InterruptedException {
    Properties props = new Properties();
    props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, VALUE_SERIALIZER);
    if (!exactlyOnce) {
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, KafkaSinglePortExactlyOnceOutputOperator.KEY_SERIALIZER);
    }
    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, getClusterConfig());
    props.put(VALUE_DESERIALIZER_CLASS_CONFIG, VALUE_DESERIALIZER);
    Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
    attributeMap.put(Context.DAGContext.APPLICATION_NAME, "MyKafkaApp");
    attributeMap.put(DAG.APPLICATION_PATH, APPLICATION_PATH);
    OperatorContext operatorContext = mockOperatorContext(2, attributeMap);
    cleanUp(operatorContext);
    Operator kafkaOutput;
    DefaultInputPort<Person> inputPort;
    if (exactlyOnce) {
        KafkaSinglePortExactlyOnceOutputOperator<Person> kafkaOutputTemp = ResetKafkaOutput(testName, props, operatorContext);
        inputPort = kafkaOutputTemp.inputPort;
        kafkaOutput = kafkaOutputTemp;
    } else {
        KafkaSinglePortOutputOperator<String, Person> kafkaOutputTemp = ResetKafkaSimpleOutput(testName, props, operatorContext);
        inputPort = kafkaOutputTemp.inputPort;
        kafkaOutput = kafkaOutputTemp;
    }
    kafkaOutput.beginWindow(1);
    inputPort.getSink().put(toKafka.get(0));
    inputPort.getSink().put(toKafka.get(1));
    inputPort.getSink().put(toKafka.get(2));
    kafkaOutput.endWindow();
    kafkaOutput.beginWindow(2);
    inputPort.getSink().put(toKafka.get(3));
    inputPort.getSink().put(toKafka.get(4));
    inputPort.getSink().put(toKafka.get(5));
    kafkaOutput.endWindow();
    kafkaOutput.beginWindow(3);
    inputPort.getSink().put(toKafka.get(6));
    inputPort.getSink().put(toKafka.get(7));
    if (hasFailure) {
        if (exactlyOnce) {
            KafkaSinglePortExactlyOnceOutputOperator<Person> kafkaOutputTemp = ResetKafkaOutput(testName, props, operatorContext);
            inputPort = kafkaOutputTemp.inputPort;
            kafkaOutput = kafkaOutputTemp;
        } else {
            KafkaSinglePortOutputOperator<String, Person> kafkaOutputTemp = ResetKafkaSimpleOutput(testName, props, operatorContext);
            inputPort = kafkaOutputTemp.inputPort;
            kafkaOutput = kafkaOutputTemp;
        }
        kafkaOutput.beginWindow(2);
        inputPort.getSink().put(toKafka.get(3));
        inputPort.getSink().put(toKafka.get(4));
        inputPort.getSink().put(toKafka.get(5));
        kafkaOutput.endWindow();
        kafkaOutput.beginWindow(3);
        inputPort.getSink().put(toKafka.get(6));
        if (!differentTuplesAfterRecovery) {
            inputPort.getSink().put(toKafka.get(7));
        }
    }
    inputPort.getSink().put(toKafka.get(8));
    inputPort.getSink().put(toKafka.get(9));
    kafkaOutput.endWindow();
    kafkaOutput.beginWindow(4);
    inputPort.getSink().put(toKafka.get(10));
    inputPort.getSink().put(toKafka.get(11));
    kafkaOutput.endWindow();
    cleanUp(operatorContext);
}

Example 30 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class HiveMockTest method testInsertString.

@Test
public void testInsertString() throws Exception {
    HiveStore hiveStore = createStore(null);
    hiveStore.setFilepath(testdir);
    ArrayList<String> hivePartitionColumns = new ArrayList<String>();
    hivePartitionColumns.add("dt");
    hiveInitializeDatabase(createStore(null));
    HiveOperator hiveOperator = new HiveOperator();
    hiveOperator.setStore(hiveStore);
    hiveOperator.setTablename(tablename);
    hiveOperator.setHivePartitionColumns(hivePartitionColumns);
    FSRollingTestImpl fsRolling = new FSRollingTestImpl();
    fsRolling.setFilePath(testdir);
    short permission = 511;
    fsRolling.setFilePermission(permission);
    fsRolling.setAlwaysWriteToTmp(false);
    fsRolling.setMaxLength(128);
    AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
    attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_LEAST_ONCE);
    attributeMap.put(OperatorContext.ACTIVATION_WINDOW_ID, -1L);
    attributeMap.put(DAG.APPLICATION_ID, APP_ID);
    OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);
    fsRolling.setup(context);
    hiveOperator.setup(context);
    FilePartitionMapping mapping1 = new FilePartitionMapping();
    FilePartitionMapping mapping2 = new FilePartitionMapping();
    mapping1.setFilename(APP_ID + "/" + OPERATOR_ID + "/" + "2014-12-10" + "/" + "0-transaction.out.part.0");
    ArrayList<String> partitions1 = new ArrayList<String>();
    partitions1.add("2014-12-10");
    mapping1.setPartition(partitions1);
    ArrayList<String> partitions2 = new ArrayList<String>();
    partitions2.add("2014-12-11");
    mapping2.setFilename(APP_ID + "/" + OPERATOR_ID + "/" + "2014-12-11" + "/" + "0-transaction.out.part.0");
    mapping2.setPartition(partitions2);
    for (int wid = 0, total = 0; wid < NUM_WINDOWS; wid++) {
        fsRolling.beginWindow(wid);
        for (int tupleCounter = 0; tupleCounter < BLAST_SIZE && total < DATABASE_SIZE; tupleCounter++, total++) {
            fsRolling.input.process("2014-12-1" + tupleCounter);
        }
        if (wid == 7) {
            fsRolling.committed(wid - 1);
            hiveOperator.processTuple(mapping1);
            hiveOperator.processTuple(mapping2);
        }
        fsRolling.endWindow();
        if (wid == 6) {
            fsRolling.beforeCheckpoint(wid);
            fsRolling.checkpointed(wid);
        }
    }
    fsRolling.teardown();
    hiveStore.connect();
    client.execute("select * from " + tablename + " where dt='2014-12-10'");
    List<String> recordsInDatePartition1 = client.fetchAll();
    client.execute("select * from " + tablename + " where dt='2014-12-11'");
    List<String> recordsInDatePartition2 = client.fetchAll();
    client.execute("drop table " + tablename);
    hiveStore.disconnect();
    Assert.assertEquals(7, recordsInDatePartition1.size());
    for (int i = 0; i < recordsInDatePartition1.size(); i++) {
        LOG.debug("records in first date partition are {}", recordsInDatePartition1.get(i));
        /*An array containing partition and data is returned as a string record, hence we need to upcast it to an object first
       and then downcast to a string in order to use in Assert.*/
        Object record = recordsInDatePartition1.get(i);
        Object[] records = (Object[]) record;
        Assert.assertEquals("2014-12-10", records[1]);
    }
    Assert.assertEquals(7, recordsInDatePartition2.size());
    for (int i = 0; i < recordsInDatePartition2.size(); i++) {
        LOG.debug("records in second date partition are {}", recordsInDatePartition2.get(i));
        Object record = recordsInDatePartition2.get(i);
        Object[] records = (Object[]) record;
        Assert.assertEquals("2014-12-11", records[1]);
    }
}