Examples with OperatorContext com.datatorrent.api.Context.OperatorContext used on opensource projects

Example 56 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class JdbcPojoOperatorTest method testJdbcPojoInsertOutputOperatorNullId.

/**
 * This test will assume direct mapping for POJO fields to DB columns.
 * Non-Nullable DB field missing in POJO id1 field which is non-nullable in DB
 * is missing from POJO POJO(id, name) -> DB(id1, name)
 */
@Test
public void testJdbcPojoInsertOutputOperatorNullId() {
    JdbcTransactionalStore transactionalStore = new JdbcTransactionalStore();
    transactionalStore.setDatabaseDriver(DB_DRIVER);
    transactionalStore.setDatabaseUrl(URL);
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributeMap = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    attributeMap.put(DAG.APPLICATION_ID, APP_ID);
    OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);
    TestPOJOOutputOperator outputOperator = new TestPOJOOutputOperator();
    outputOperator.setBatchSize(3);
    outputOperator.setTablename(TABLE_POJO_NAME_ID_DIFF);
    outputOperator.setStore(transactionalStore);
    outputOperator.setup(context);
    Attribute.AttributeMap.DefaultAttributeMap portAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
    portAttributes.put(Context.PortContext.TUPLE_CLASS, TestPOJOEvent.class);
    TestPortContext tpc = new TestPortContext(portAttributes);
    outputOperator.input.setup(tpc);
    boolean exceptionOccurred = false;
    try {
        outputOperator.activate(context);
    } catch (Exception e) {
        exceptionOccurred = true;
        Assert.assertTrue(e instanceof RuntimeException);
        Assert.assertTrue(e.getMessage().toLowerCase().contains("id1 not found in pojo"));
    }
    Assert.assertTrue(exceptionOccurred);
}

Example 57 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class JdbcPojoOperatorTest method testJdbcPojoOutputOperatorMerge.

@Test
public void testJdbcPojoOutputOperatorMerge() {
    JdbcTransactionalStore transactionalStore = new JdbcTransactionalStore();
    transactionalStore.setDatabaseDriver(DB_DRIVER);
    transactionalStore.setDatabaseUrl(URL);
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributeMap = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    attributeMap.put(DAG.APPLICATION_ID, APP_ID);
    OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);
    TestPOJOOutputOperator.TestPOJONonInsertOutputOperator updateOperator = new TestPOJOOutputOperator.TestPOJONonInsertOutputOperator();
    updateOperator.setBatchSize(3);
    updateOperator.setStore(transactionalStore);
    updateOperator.setSqlStatement("MERGE INTO " + TABLE_POJO_NAME + " AS T USING (VALUES (?, ?)) AS FOO(id, name) " + "ON T.id = FOO.id " + "WHEN MATCHED THEN UPDATE SET name = FOO.name " + "WHEN NOT MATCHED THEN INSERT( id, name ) VALUES (FOO.id, FOO.name);");
    List<JdbcFieldInfo> fieldInfos = Lists.newArrayList();
    fieldInfos.add(new JdbcFieldInfo("id", "id", null, Types.INTEGER));
    fieldInfos.add(new JdbcFieldInfo("name", "name", null, Types.VARCHAR));
    updateOperator.setFieldInfos(fieldInfos);
    updateOperator.setup(context);
    Attribute.AttributeMap.DefaultAttributeMap portAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
    portAttributes.put(Context.PortContext.TUPLE_CLASS, TestPOJOEvent.class);
    TestPortContext tpc = new TestPortContext(portAttributes);
    updateOperator.input.setup(tpc);
    updateOperator.activate(context);
    List<TestPOJOEvent> events = Lists.newArrayList();
    for (int i = 0; i < 10; i++) {
        events.add(new TestPOJOEvent(i, "test" + i));
    }
    for (int i = 0; i < 5; i++) {
        events.add(new TestPOJOEvent(i, "test" + 100));
    }
    updateOperator.getDistinctNonUnique();
    updateOperator.beginWindow(0);
    for (TestPOJOEvent event : events) {
        updateOperator.input.process(event);
    }
    updateOperator.endWindow();
    // Expect 10 unique ids: 0 - 9
    Assert.assertEquals("rows in db", 10, updateOperator.getNumOfEventsInStore());
    // Expect 6 unique name: test-100, test-5, test-6, test-7, test-8, test-9
    Assert.assertEquals("rows in db", 6, updateOperator.getDistinctNonUnique());
}

Example 58 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class JdbcPojoOperatorTest method testJdbcPojoInsertOutputOperatorNullName.

/**
 * This test will assume direct mapping for POJO fields to DB columns Nullable
 * DB field missing in POJO name1 field, which is nullable in DB is missing
 * from POJO POJO(id, name) -> DB(id, name1)
 */
@Test
public void testJdbcPojoInsertOutputOperatorNullName() {
    JdbcTransactionalStore transactionalStore = new JdbcTransactionalStore();
    transactionalStore.setDatabaseDriver(DB_DRIVER);
    transactionalStore.setDatabaseUrl(URL);
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributeMap = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    attributeMap.put(DAG.APPLICATION_ID, APP_ID);
    OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);
    TestPOJOOutputOperator outputOperator = new TestPOJOOutputOperator();
    outputOperator.setBatchSize(3);
    outputOperator.setTablename(TABLE_POJO_NAME_NAME_DIFF);
    outputOperator.setStore(transactionalStore);
    outputOperator.setup(context);
    Attribute.AttributeMap.DefaultAttributeMap portAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
    portAttributes.put(Context.PortContext.TUPLE_CLASS, TestPOJOEvent.class);
    TestPortContext tpc = new TestPortContext(portAttributes);
    outputOperator.input.setup(tpc);
    outputOperator.activate(context);
    List<TestPOJOEvent> events = Lists.newArrayList();
    for (int i = 0; i < 10; i++) {
        events.add(new TestPOJOEvent(i, "test" + i));
    }
    outputOperator.beginWindow(0);
    for (TestPOJOEvent event : events) {
        outputOperator.input.process(event);
    }
    outputOperator.endWindow();
    Assert.assertEquals("rows in db", 10, outputOperator.getNumOfEventsInStore(TABLE_POJO_NAME_NAME_DIFF));
    Assert.assertEquals("null name rows in db", 10, outputOperator.getNumOfNullEventsInStore(TABLE_POJO_NAME_NAME_DIFF));
}

Example 59 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class DeduperBoundedPOJOImplTest method testDedup.

@Test
public void testDedup() {
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributes = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    attributes.put(DAG.APPLICATION_ID, APP_ID);
    attributes.put(DAG.APPLICATION_PATH, applicationPath);
    attributes.put(DAG.InputPortMeta.TUPLE_CLASS, TestPojo.class);
    OperatorContext context = mockOperatorContext(OPERATOR_ID, attributes);
    deduper.setup(context);
    deduper.input.setup(new PortContext(attributes, context));
    deduper.activate(context);
    CollectorTestSink<TestPojo> uniqueSink = new CollectorTestSink<TestPojo>();
    TestUtils.setSink(deduper.unique, uniqueSink);
    CollectorTestSink<TestPojo> duplicateSink = new CollectorTestSink<TestPojo>();
    TestUtils.setSink(deduper.duplicate, duplicateSink);
    CollectorTestSink<TestPojo> expiredSink = new CollectorTestSink<TestPojo>();
    TestUtils.setSink(deduper.expired, expiredSink);
    deduper.beginWindow(0);
    Random r = new Random();
    int k = 1;
    for (int i = 1; i <= 1000; i++) {
        TestPojo pojo = new TestPojo(i, new Date(), k++);
        deduper.input.process(pojo);
        if (i % 10 == 0) {
            int dupId = r.nextInt(i);
            TestPojo pojoDuplicate = new TestPojo(dupId == 0 ? 1 : dupId, new Date(), k++);
            deduper.input.process(pojoDuplicate);
        }
    }
    deduper.handleIdleTime();
    deduper.endWindow();
    Assert.assertTrue(uniqueSink.collectedTuples.size() == 1000);
    Assert.assertTrue(duplicateSink.collectedTuples.size() == 100);
    deduper.teardown();
}

Example 60 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class JDBCLookupCacheBackedOperatorTest method setup.

@BeforeClass
public static void setup() throws Exception {
    // This will load the JDBC driver, each DB has its own driver
    Class.forName(INMEM_DB_DRIVER).newInstance();
    Connection con = DriverManager.getConnection(INMEM_DB_URL);
    Statement stmt = con.createStatement();
    String createTable = "CREATE TABLE IF NOT EXISTS " + TABLE_NAME + " (col1 INTEGER, col2 VARCHAR(20))";
    stmt.executeUpdate(createTable);
    stmt.executeUpdate("Delete from " + TABLE_NAME);
    // populate the database
    for (Map.Entry<Integer, String> entry : mapping.entrySet()) {
        String insert = "INSERT INTO " + TABLE_NAME + " (col1, col2) VALUES (" + entry.getKey() + ", '" + entry.getValue() + "')";
        stmt.executeUpdate(insert);
    }
    // Setup the operator
    lookupCacheBackedOperator.getStore().setDatabaseUrl(INMEM_DB_URL);
    lookupCacheBackedOperator.getStore().setDatabaseDriver(INMEM_DB_DRIVER);
    Calendar now = Calendar.getInstance();
    now.add(Calendar.SECOND, 5);
    SimpleDateFormat format = new SimpleDateFormat("HH:mm:ss");
    lookupCacheBackedOperator.getCacheManager().setRefreshTime(format.format(now.getTime()));
    lookupCacheBackedOperator.output.setSink(sink);
    OperatorContext context = mockOperatorContext(7);
    lookupCacheBackedOperator.setup(context);
}