Examples with OperatorContext com.datatorrent.api.Context.OperatorContext used on opensource projects

Example 41 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class DeduperBoundedPOJOImplTest method testDedup.

@Test
public void testDedup() {
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributes = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    attributes.put(DAG.APPLICATION_ID, APP_ID);
    attributes.put(DAG.APPLICATION_PATH, applicationPath);
    attributes.put(DAG.InputPortMeta.TUPLE_CLASS, TestPojo.class);
    OperatorContext context = mockOperatorContext(OPERATOR_ID, attributes);
    deduper.setup(context);
    deduper.input.setup(new PortContext(attributes, context));
    deduper.activate(context);
    CollectorTestSink<TestPojo> uniqueSink = new CollectorTestSink<TestPojo>();
    TestUtils.setSink(deduper.unique, uniqueSink);
    CollectorTestSink<TestPojo> duplicateSink = new CollectorTestSink<TestPojo>();
    TestUtils.setSink(deduper.duplicate, duplicateSink);
    CollectorTestSink<TestPojo> expiredSink = new CollectorTestSink<TestPojo>();
    TestUtils.setSink(deduper.expired, expiredSink);
    deduper.beginWindow(0);
    Random r = new Random();
    int k = 1;
    for (int i = 1; i <= 1000; i++) {
        TestPojo pojo = new TestPojo(i, new Date(), k++);
        deduper.input.process(pojo);
        if (i % 10 == 0) {
            int dupId = r.nextInt(i);
            TestPojo pojoDuplicate = new TestPojo(dupId == 0 ? 1 : dupId, new Date(), k++);
            deduper.input.process(pojoDuplicate);
        }
    }
    deduper.handleIdleTime();
    deduper.endWindow();
    Assert.assertTrue(uniqueSink.collectedTuples.size() == 1000);
    Assert.assertTrue(duplicateSink.collectedTuples.size() == 100);
    deduper.teardown();
}

Example 42 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class DeduperTimeBasedPOJOImplTest method testDedupDifferentWindowSameKey.

@Test
public void testDedupDifferentWindowSameKey() {
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributes = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    attributes.put(DAG.APPLICATION_ID, APP_ID);
    attributes.put(DAG.APPLICATION_PATH, applicationPath);
    attributes.put(DAG.InputPortMeta.TUPLE_CLASS, TestPojo.class);
    OperatorContext context = mockOperatorContext(OPERATOR_ID, attributes);
    deduper.setup(context);
    deduper.input.setup(new PortContext(attributes, context));
    deduper.activate(context);
    CollectorTestSink<TestPojo> uniqueSink = new CollectorTestSink<TestPojo>();
    TestUtils.setSink(deduper.unique, uniqueSink);
    CollectorTestSink<TestPojo> duplicateSink = new CollectorTestSink<TestPojo>();
    TestUtils.setSink(deduper.duplicate, duplicateSink);
    CollectorTestSink<TestPojo> expiredSink = new CollectorTestSink<TestPojo>();
    TestUtils.setSink(deduper.expired, expiredSink);
    deduper.beginWindow(0);
    long millis = System.currentTimeMillis();
    deduper.input.process(new TestPojo(10, new Date(millis)));
    deduper.input.process(new TestPojo(11, new Date(millis + 10000)));
    deduper.input.process(new TestPojo(12, new Date(millis + 20000)));
    deduper.input.process(new TestPojo(13, new Date(millis + 30000)));
    deduper.input.process(new TestPojo(14, new Date(millis + 40000)));
    deduper.input.process(new TestPojo(15, new Date(millis + 50000)));
    // Duplicate
    deduper.input.process(new TestPojo(10, new Date(millis)));
    deduper.input.process(new TestPojo(16, new Date(millis + 60000)));
    // New tuple with same key but outside expired window.
    deduper.input.process(new TestPojo(10, new Date(millis + 70000)));
    // Earlier tuple with earlier time -- Expired
    deduper.input.process(new TestPojo(10, new Date(millis)));
    // New tuple repeated again - Duplicate
    deduper.input.process(new TestPojo(10, new Date(millis + 70000)));
    deduper.handleIdleTime();
    deduper.endWindow();
    Assert.assertTrue(uniqueSink.collectedTuples.size() == 8);
    Assert.assertTrue(duplicateSink.collectedTuples.size() == 2);
    Assert.assertTrue(expiredSink.collectedTuples.size() == 1);
    deduper.teardown();
}

Example 43 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class JDBCLookupCacheBackedOperatorTest method setup.

@BeforeClass
public static void setup() throws Exception {
    // This will load the JDBC driver, each DB has its own driver
    Class.forName(INMEM_DB_DRIVER).newInstance();
    Connection con = DriverManager.getConnection(INMEM_DB_URL);
    Statement stmt = con.createStatement();
    String createTable = "CREATE TABLE IF NOT EXISTS " + TABLE_NAME + " (col1 INTEGER, col2 VARCHAR(20))";
    stmt.executeUpdate(createTable);
    stmt.executeUpdate("Delete from " + TABLE_NAME);
    // populate the database
    for (Map.Entry<Integer, String> entry : mapping.entrySet()) {
        String insert = "INSERT INTO " + TABLE_NAME + " (col1, col2) VALUES (" + entry.getKey() + ", '" + entry.getValue() + "')";
        stmt.executeUpdate(insert);
    }
    // Setup the operator
    lookupCacheBackedOperator.getStore().setDatabaseUrl(INMEM_DB_URL);
    lookupCacheBackedOperator.getStore().setDatabaseDriver(INMEM_DB_DRIVER);
    Calendar now = Calendar.getInstance();
    now.add(Calendar.SECOND, 5);
    SimpleDateFormat format = new SimpleDateFormat("HH:mm:ss");
    lookupCacheBackedOperator.getCacheManager().setRefreshTime(format.format(now.getTime()));
    lookupCacheBackedOperator.output.setSink(sink);
    OperatorContext context = mockOperatorContext(7);
    lookupCacheBackedOperator.setup(context);
}

Example 44 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class JdbcNonTransactionalBatchOutputOperatorTest method testAtMostOnceFullBatch.

@Test
public void testAtMostOnceFullBatch() {
    JdbcOperatorTest.cleanTable();
    Random random = new Random();
    TestOutputOperator outputOperator = createOperator(ProcessingMode.AT_MOST_ONCE);
    outputOperator.beginWindow(0);
    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
        outputOperator.input.put(new TestEvent(random.nextInt()));
    }
    outputOperator.endWindow();
    Assert.assertEquals("Commit window id ", 0, outputOperator.getStore().getCommittedWindowId(APP_ID, OPERATOR_ID));
    Assert.assertEquals("Batch should be written", BATCH_SIZE, outputOperator.getNumOfEventsInStore(outputOperator.getStore().connection));
    outputOperator.beginWindow(1);
    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
        outputOperator.input.put(new TestEvent(random.nextInt()));
    }
    Assert.assertEquals("Commit window id ", 0, outputOperator.getStore().getCommittedWindowId(APP_ID, OPERATOR_ID));
    Assert.assertEquals("Batch should be written", 2 * BATCH_SIZE, outputOperator.getNumOfEventsInStore(outputOperator.getStore().connection));
    outputOperator.getStore().disconnect();
    // //
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributeMap = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_MOST_ONCE);
    attributeMap.put(OperatorContext.ACTIVATION_WINDOW_ID, 0L);
    attributeMap.put(DAG.APPLICATION_ID, APP_ID);
    OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);
    outputOperator.setup(context);
    outputOperator.beginWindow(2);
    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
        outputOperator.input.put(new TestEvent(random.nextInt()));
    }
    outputOperator.endWindow();
    Assert.assertEquals("Commit window id ", 2, outputOperator.getStore().getCommittedWindowId(APP_ID, OPERATOR_ID));
    Assert.assertEquals("Batch should be written", 3 * BATCH_SIZE, outputOperator.getNumOfEventsInStore(outputOperator.getStore().connection));
}

Example 45 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class JdbcNonTransactionalBatchOutputOperatorTest method createOperator.

private static TestOutputOperator createOperator(ProcessingMode processingMode) {
    JdbcNonTransactionalStore store = new JdbcNonTransactionalStore();
    store.setDatabaseDriver(JdbcNonTransactionalOutputOperatorTest.DB_DRIVER);
    store.setDatabaseUrl(JdbcNonTransactionalOutputOperatorTest.URL);
    TestOutputOperator outputOperator = new TestOutputOperator();
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributeMap = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    attributeMap.put(OperatorContext.PROCESSING_MODE, processingMode);
    attributeMap.put(OperatorContext.ACTIVATION_WINDOW_ID, -1L);
    attributeMap.put(DAG.APPLICATION_ID, APP_ID);
    OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);
    outputOperator.setStore(store);
    outputOperator.setBatchSize(BATCH_SIZE);
    outputOperator.setup(context);
    return outputOperator;
}