Examples with OperatorContext com.datatorrent.api.Context.OperatorContext used on opensource projects

Example 11 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class RedisInputOperatorTest method testRecoveryAndIdempotency.

@Test
public void testRecoveryAndIdempotency() throws Exception {
    this.operatorStore = new RedisStore();
    this.testStore = new RedisStore();
    testStore.connect();
    ScanParams params = new ScanParams();
    params.count(1);
    testStore.put("test_abc", "789");
    testStore.put("test_def", "456");
    testStore.put("test_ghi", "123");
    RedisKeyValueInputOperator operator = new RedisKeyValueInputOperator();
    operator.setWindowDataManager(new FSWindowDataManager());
    operator.setStore(operatorStore);
    operator.setScanCount(1);
    Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
    CollectorTestSink<Object> sink = new CollectorTestSink<Object>();
    operator.outputPort.setSink(sink);
    OperatorContext context = mockOperatorContext(1, attributeMap);
    try {
        operator.setup(context);
        operator.beginWindow(1);
        operator.emitTuples();
        operator.endWindow();
        int numberOfMessagesInWindow1 = sink.collectedTuples.size();
        sink.collectedTuples.clear();
        operator.beginWindow(2);
        operator.emitTuples();
        operator.endWindow();
        int numberOfMessagesInWindow2 = sink.collectedTuples.size();
        sink.collectedTuples.clear();
        // failure and then re-deployment of operator
        // Re-instantiating to reset values
        operator = new RedisKeyValueInputOperator();
        operator.setWindowDataManager(new FSWindowDataManager());
        operator.setStore(operatorStore);
        operator.setScanCount(1);
        operator.outputPort.setSink(sink);
        operator.setup(context);
        Assert.assertEquals("largest recovery window", 2, operator.getWindowDataManager().getLargestCompletedWindow());
        operator.beginWindow(1);
        operator.emitTuples();
        operator.emitTuples();
        operator.endWindow();
        Assert.assertEquals("num of messages in window 1", numberOfMessagesInWindow1, sink.collectedTuples.size());
        sink.collectedTuples.clear();
        operator.beginWindow(2);
        operator.emitTuples();
        operator.endWindow();
        Assert.assertEquals("num of messages in window 2", numberOfMessagesInWindow2, sink.collectedTuples.size());
    } finally {
        for (Object e : sink.collectedTuples) {
            KeyValPair<String, String> entry = (KeyValPair<String, String>) e;
            testStore.remove(entry.getKey());
        }
        sink.collectedTuples.clear();
        operator.getWindowDataManager().committed(5);
        operator.teardown();
    }
}

Example 12 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class RabbitMQInputOperatorTest method testRecoveryAndIdempotency.

@Test
public void testRecoveryAndIdempotency() throws Exception {
    RabbitMQInputOperator operator = new RabbitMQInputOperator();
    operator.setWindowDataManager(new FSWindowDataManager());
    operator.setHost("localhost");
    operator.setExchange("testEx");
    operator.setExchangeType("fanout");
    Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
    CollectorTestSink<Object> sink = new CollectorTestSink<Object>();
    operator.outputPort.setSink(sink);
    OperatorContext context = mockOperatorContext(1, attributeMap);
    operator.setup(context);
    operator.activate(context);
    final RabbitMQMessageGenerator publisher = new RabbitMQMessageGenerator();
    publisher.setup();
    publisher.generateMessages(5);
    Thread.sleep(10000);
    operator.beginWindow(1);
    operator.emitTuples();
    operator.endWindow();
    operator.deactivate();
    Assert.assertEquals("num of messages in window 1", 15, sink.collectedTuples.size());
    // failure and then re-deployment of operator
    sink.collectedTuples.clear();
    operator.setup(context);
    operator.activate(context);
    Assert.assertEquals("largest recovery window", 1, operator.getWindowDataManager().getLargestCompletedWindow());
    operator.beginWindow(1);
    operator.endWindow();
    Assert.assertEquals("num of messages in window 1", 15, sink.collectedTuples.size());
    sink.collectedTuples.clear();
    operator.deactivate();
    operator.teardown();
    operator.getWindowDataManager().committed(1);
    publisher.teardown();
}

Example 13 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class CassandraLookupCacheBackedOperatorTest method setupDB.

@BeforeClass
public static void setupDB() {
    try {
        // This will load the JDBC driver, each DB has its own driver
        Class.forName(CASSANDRA_DB_DRIVER).newInstance();
        Connection con = DriverManager.getConnection(CASSANDRA_DB_URL);
        Statement stmt = con.createStatement();
        String useSystem = " USE system";
        stmt.executeUpdate(useSystem);
        ResultSet rs = stmt.executeQuery("SELECT * FROM schema_keyspaces");
        boolean foundTest = false;
        while (rs.next()) {
            if (KEYSPACE_NAME.equals(rs.getString(1))) {
                foundTest = true;
            }
        }
        if (!foundTest) {
            String createKeyspace = "CREATE KEYSPACE " + KEYSPACE_NAME + " WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':3}";
            stmt.executeUpdate(createKeyspace);
        }
        String useKeyspace = "USE " + KEYSPACE_NAME;
        stmt.executeUpdate(useKeyspace);
        String createTable = "CREATE TABLE " + TABLE_NAME + " (col1 int primary key, col2 varchar)";
        stmt.executeUpdate(createTable);
        // populate the database
        for (Map.Entry<Integer, String> entry : mapping.entrySet()) {
            String insert = "INSERT INTO " + TABLE_NAME + " (col1, col2) values (" + entry.getKey() + ", '" + entry.getValue() + "')";
            stmt.executeUpdate(insert);
        }
        // Setup the operator
        lookupCacheBackedOperator.getStore().setDatabaseUrl(CASSANDRA_DB_URL + "/" + KEYSPACE_NAME);
        lookupCacheBackedOperator.getStore().setDatabaseDriver(CASSANDRA_DB_DRIVER);
        Calendar now = Calendar.getInstance(TimeZone.getTimeZone("PST"));
        now.add(Calendar.SECOND, 15);
        SimpleDateFormat format = new SimpleDateFormat("HH:mm:ss z");
        lookupCacheBackedOperator.getCacheManager().setRefreshTime(format.format(now.getTime()));
        lookupCacheBackedOperator.output.setSink(sink);
        OperatorContext context = mockOperatorContext(7);
        lookupCacheBackedOperator.setup(context);
    } catch (Exception ex) {
        logger.error("cassandra setup", ex);
    }
}

Example 14 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class CassandraOperatorTest method testCassandraInputOperator.

/*
   * This test can be run on cassandra server installed on node17.
   */
@Test
public void testCassandraInputOperator() {
    String query1 = "SELECT * FROM " + KEYSPACE + "." + "%t;";
    CassandraStore store = new CassandraStore();
    store.setNode(NODE);
    store.setKeyspace(KEYSPACE);
    AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
    attributeMap.put(DAG.APPLICATION_ID, APP_ID);
    OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);
    TestInputOperator inputOperator = new TestInputOperator();
    inputOperator.setStore(store);
    inputOperator.setQuery(query1);
    inputOperator.setTablename(TABLE_NAME_INPUT);
    inputOperator.setPrimaryKeyColumn("id");
    List<FieldInfo> fieldInfos = Lists.newArrayList();
    fieldInfos.add(new FieldInfo("id", "id", null));
    fieldInfos.add(new FieldInfo("age", "age", null));
    fieldInfos.add(new FieldInfo("lastname", "lastname", null));
    inputOperator.setFieldInfos(fieldInfos);
    inputOperator.insertEventsInTable(30);
    CollectorTestSink<Object> sink = new CollectorTestSink<>();
    inputOperator.outputPort.setSink(sink);
    Attribute.AttributeMap.DefaultAttributeMap portAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
    portAttributes.put(Context.PortContext.TUPLE_CLASS, TestInputPojo.class);
    TestPortContext tpc = new TestPortContext(portAttributes);
    inputOperator.setup(context);
    inputOperator.outputPort.setup(tpc);
    inputOperator.activate(context);
    inputOperator.beginWindow(0);
    inputOperator.emitTuples();
    inputOperator.endWindow();
    Assert.assertEquals("rows from db", 30, sink.collectedTuples.size());
    ArrayList<Integer> listOfIDs = inputOperator.getIds();
    // Rows are not stored in the same order in cassandra table in which they are inserted.
    for (int i = 0; i < 10; i++) {
        TestInputPojo object = (TestInputPojo) sink.collectedTuples.get(i);
        Assert.assertTrue("id set in testpojo", listOfIDs.contains(object.getId()));
        Assert.assertEquals("name set in testpojo", inputOperator.getNames().get(object.getId()), object.getLastname());
        Assert.assertEquals("age set in testpojo", inputOperator.getAge().get(object.getId()).intValue(), object.getAge());
    }
    sink.clear();
    inputOperator.columnDataTypes.clear();
    String query2 = "SELECT * FROM " + KEYSPACE + "." + "%t where token(%p) > %v;";
    inputOperator.setQuery(query2);
    inputOperator.setStartRow(10);
    inputOperator.setup(context);
    inputOperator.outputPort.setup(tpc);
    inputOperator.activate(context);
    inputOperator.beginWindow(1);
    inputOperator.emitTuples();
    inputOperator.endWindow();
    Assert.assertEquals("rows from db", 26, sink.collectedTuples.size());
    sink.clear();
    inputOperator.columnDataTypes.clear();
    String query3 = "SELECT * FROM " + KEYSPACE + "." + "%t where token(%p) > %v LIMIT %l;";
    inputOperator.setQuery(query3);
    inputOperator.setStartRow(1);
    inputOperator.setLimit(10);
    inputOperator.setup(context);
    inputOperator.outputPort.setup(tpc);
    inputOperator.activate(context);
    inputOperator.beginWindow(2);
    inputOperator.emitTuples();
    inputOperator.endWindow();
    Assert.assertEquals("rows from db", 10, sink.collectedTuples.size());
}

Example 15 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class HBasePOJOPutOperatorTest method setupOperator.

protected void setupOperator(HBasePOJOPutOperator operator) {
    configure(operator);
    AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
    attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_LEAST_ONCE);
    attributeMap.put(OperatorContext.ACTIVATION_WINDOW_ID, -1L);
    attributeMap.put(DAG.APPLICATION_ID, APP_ID);
    OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);
    operator.setup(context);
}