Examples with OperatorContext com.datatorrent.api.Context.OperatorContext used on opensource projects

Example 46 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class SpillableSetMultimapImplTest method testLoad.

@Test
public void testLoad() {
    Random random = new Random();
    final int keySize = 1000000;
    final int valueSize = 100000000;
    final int numOfEntry = 100000;
    SpillableStateStore store = testMeta.store;
    SpillableSetMultimapImpl<String, String> multimap = new SpillableSetMultimapImpl<>(testMeta.store, ID1, 0L, createStringSerde(), createStringSerde());
    Attribute.AttributeMap.DefaultAttributeMap attributes = new Attribute.AttributeMap.DefaultAttributeMap();
    attributes.put(DAG.APPLICATION_PATH, testMeta.applicationPath);
    OperatorContext context = mockOperatorContext(testMeta.operatorContext.getId(), attributes);
    store.setup(context);
    multimap.setup(context);
    store.beginWindow(1);
    multimap.beginWindow(1);
    for (int i = 0; i < numOfEntry; ++i) {
        multimap.put(String.valueOf(random.nextInt(keySize)), String.valueOf(random.nextInt(valueSize)));
    }
    multimap.endWindow();
    store.endWindow();
}

Example 47 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class WindowUtilsTest method msToAppWindowCountRoundingTest.

@Test
public void msToAppWindowCountRoundingTest() {
    OperatorContext context = createOperatorContext(500, 10);
    long appWindowCount = WindowUtils.msToAppWindowCount(context, 10001L);
    Assert.assertEquals(3, appWindowCount);
}

Example 48 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class DeduperTimeBasedPOJOImplTest method testDedupDifferentWindowSameKey.

@Test
public void testDedupDifferentWindowSameKey() {
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributes = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    attributes.put(DAG.APPLICATION_ID, APP_ID);
    attributes.put(DAG.APPLICATION_PATH, applicationPath);
    attributes.put(DAG.InputPortMeta.TUPLE_CLASS, TestPojo.class);
    OperatorContext context = mockOperatorContext(OPERATOR_ID, attributes);
    deduper.setup(context);
    deduper.input.setup(new PortContext(attributes, context));
    deduper.activate(context);
    CollectorTestSink<TestPojo> uniqueSink = new CollectorTestSink<TestPojo>();
    TestUtils.setSink(deduper.unique, uniqueSink);
    CollectorTestSink<TestPojo> duplicateSink = new CollectorTestSink<TestPojo>();
    TestUtils.setSink(deduper.duplicate, duplicateSink);
    CollectorTestSink<TestPojo> expiredSink = new CollectorTestSink<TestPojo>();
    TestUtils.setSink(deduper.expired, expiredSink);
    deduper.beginWindow(0);
    long millis = System.currentTimeMillis();
    deduper.input.process(new TestPojo(10, new Date(millis)));
    deduper.input.process(new TestPojo(11, new Date(millis + 10000)));
    deduper.input.process(new TestPojo(12, new Date(millis + 20000)));
    deduper.input.process(new TestPojo(13, new Date(millis + 30000)));
    deduper.input.process(new TestPojo(14, new Date(millis + 40000)));
    deduper.input.process(new TestPojo(15, new Date(millis + 50000)));
    // Duplicate
    deduper.input.process(new TestPojo(10, new Date(millis)));
    deduper.input.process(new TestPojo(16, new Date(millis + 60000)));
    // New tuple with same key but outside expired window.
    deduper.input.process(new TestPojo(10, new Date(millis + 70000)));
    // Earlier tuple with earlier time -- Expired
    deduper.input.process(new TestPojo(10, new Date(millis)));
    // New tuple repeated again - Duplicate
    deduper.input.process(new TestPojo(10, new Date(millis + 70000)));
    deduper.handleIdleTime();
    deduper.endWindow();
    Assert.assertTrue(uniqueSink.collectedTuples.size() == 8);
    Assert.assertTrue(duplicateSink.collectedTuples.size() == 2);
    Assert.assertTrue(expiredSink.collectedTuples.size() == 1);
    deduper.teardown();
}

Example 49 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class OperatorContextTestHelper method mockOperatorContext.

public static OperatorContext mockOperatorContext(int id, final AttributeMap map) {
    OperatorContext context = Mockito.mock(OperatorContext.class);
    Mockito.when(context.getId()).thenReturn(id);
    Mockito.when(context.getAttributes()).thenReturn(map);
    Mockito.doThrow(new UnsupportedOperationException("not supported")).when(context).sendMetrics(Mockito.<Collection<String>>any());
    Mockito.doThrow(new UnsupportedOperationException("not supported")).when(context).setCounters(Mockito.any());
    Mockito.when(context.getValue(Mockito.<Attribute>any())).thenAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            final Attribute key = (Attribute) invocation.getArguments()[0];
            Object value = map.get(key);
            if (value != null) {
                return value;
            }
            return key.defaultValue;
        }
    });
    Mockito.doNothing().when(context).setCounters(Mockito.any());
    Mockito.when(context.getWindowsFromCheckpoint()).thenReturn(0);
    return context;
}

Example 50 with OperatorContext

use of com.datatorrent.api.Context.OperatorContext in project apex-malhar by apache.

the class ManagedTimeStateImplTest method testRecovery.

@Test
public void testRecovery() throws ExecutionException, InterruptedException {
    Slice one = ManagedStateTestUtils.getSliceFor("1");
    testMeta.managedState.setup(testMeta.operatorContext);
    long time = System.currentTimeMillis();
    testMeta.managedState.beginWindow(0);
    testMeta.managedState.put(0, time, one, one);
    testMeta.managedState.endWindow();
    testMeta.managedState.beforeCheckpoint(0);
    testMeta.managedState.teardown();
    // there is a failure and the operator is re-deployed.
    testMeta.managedState.setStateTracker(new StateTracker());
    Attribute.AttributeMap.DefaultAttributeMap attributes = new Attribute.AttributeMap.DefaultAttributeMap();
    attributes.put(DAG.APPLICATION_PATH, testMeta.applicationPath);
    attributes.put(Context.OperatorContext.ACTIVATION_WINDOW_ID, 0L);
    OperatorContext operatorContext = mockOperatorContext(1, attributes);
    testMeta.managedState.setup(operatorContext);
    Bucket.DefaultBucket defaultBucket = (Bucket.DefaultBucket) testMeta.managedState.getBucket(0);
    Assert.assertEquals("value of one", one, defaultBucket.get(one, time, Bucket.ReadSource.MEMORY));
}