Examples with SpillableWindowedKeyedStorage org.apache.apex.malhar.lib.window.impl.SpillableWindowedKeyedStorage used on opensource projects

Example 1 with SpillableWindowedKeyedStorage

use of org.apache.apex.malhar.lib.window.impl.SpillableWindowedKeyedStorage in project apex-malhar by apache.

the class KeyedWindowedOperatorBenchmarkApp method createRetractionStorage.

@Override
protected WindowedStorage createRetractionStorage(SpillableComplexComponentImpl sccImpl) {
    if (useInMemoryStorage) {
        return new InMemoryWindowedKeyedStorage();
    }
    SpillableWindowedKeyedStorage retractionStorage = new SpillableWindowedKeyedStorage();
    retractionStorage.setSpillableComplexComponent(sccImpl);
    return retractionStorage;
}

Example 2 with SpillableWindowedKeyedStorage

use of org.apache.apex.malhar.lib.window.impl.SpillableWindowedKeyedStorage in project apex-malhar by apache.

the class SpillableWindowedStorageTest method testWindowedKeyedStorage.

@Test
public void testWindowedKeyedStorage() {
    SpillableComplexComponentImpl sccImpl = new SpillableComplexComponentImpl(testMeta.timeStore);
    SpillableWindowedKeyedStorage<String, Integer> storage = new SpillableWindowedKeyedStorage<>();
    Window window1 = new Window.TimeWindow<>(BASETIME + 1000, 10);
    Window window2 = new Window.TimeWindow<>(BASETIME + 1010, 10);
    Window window3 = new Window.TimeWindow<>(BASETIME + 1020, 10);
    storage.setSpillableComplexComponent(sccImpl);
    /*
     * storage.setup() will create Spillable Data Structures
     * storage.getSpillableComplexComponent().setup() will setup these Data Structures.
     * So storage.setup() should be called before storage.getSpillableComplexComponent().setup()
     */
    storage.setup(testMeta.operatorContext);
    storage.getSpillableComplexComponent().setup(testMeta.operatorContext);
    sccImpl.beginWindow(1000);
    storage.put(window1, "x", 1);
    storage.put(window2, "x", 2);
    storage.put(window3, "x", 3);
    sccImpl.endWindow();
    sccImpl.beginWindow(1001);
    storage.put(window1, "x", 4);
    storage.put(window2, "x", 5);
    sccImpl.endWindow();
    sccImpl.beforeCheckpoint(1001);
    SpillableWindowedKeyedStorage<String, Integer> clonedStorage = KryoCloneUtils.cloneObject(storage);
    sccImpl.checkpointed(1001);
    sccImpl.beginWindow(1002);
    storage.put(window1, "x", 6);
    storage.put(window2, "x", 7);
    storage.put(window2, "y", 8);
    sccImpl.endWindow();
    Assert.assertEquals(6L, storage.get(window1, "x").longValue());
    Assert.assertEquals(7L, storage.get(window2, "x").longValue());
    Assert.assertEquals(3L, storage.get(window3, "x").longValue());
    Assert.assertEquals(8L, storage.get(window2, "y").longValue());
    // simulating crash here
    storage.teardown();
    storage.getSpillableComplexComponent().teardown();
    storage = clonedStorage;
    testMeta.operatorContext.getAttributes().put(Context.OperatorContext.ACTIVATION_WINDOW_ID, 1001L);
    storage.getSpillableComplexComponent().setup(testMeta.operatorContext);
    storage.setup(testMeta.operatorContext);
    // recovery at window 1002
    sccImpl.beginWindow(1002);
    Assert.assertEquals(4L, storage.get(window1, "x").longValue());
    Assert.assertEquals(5L, storage.get(window2, "x").longValue());
    Assert.assertEquals(3L, storage.get(window3, "x").longValue());
    Assert.assertNull(storage.get(window2, "y"));
}

Example 3 with SpillableWindowedKeyedStorage

use of org.apache.apex.malhar.lib.window.impl.SpillableWindowedKeyedStorage in project apex-malhar by apache.

the class WindowedOperatorTest method createDefaultKeyedWindowedOperator.

private KeyedWindowedOperatorImpl<String, Long, MutableLong, Long> createDefaultKeyedWindowedOperator(boolean forSession) {
    KeyedWindowedOperatorImpl<String, Long, MutableLong, Long> windowedOperator = new KeyedWindowedOperatorImpl<>();
    if (useSpillable) {
        sccImpl = new SpillableComplexComponentImpl(testMeta.timeStore);
        // TODO: We don't yet support Spillable data structures for window state storage because SpillableMapImpl does not yet support iterating over all keys.
        windowStateStorage = new InMemoryWindowedStorage<>();
        if (forSession) {
            SpillableSessionWindowedStorage<String, MutableLong> sws = new SpillableSessionWindowedStorage<>();
            sws.setSpillableComplexComponent(sccImpl);
            keyedDataStorage = sws;
        } else {
            SpillableWindowedKeyedStorage<String, MutableLong> kds = new SpillableWindowedKeyedStorage<>();
            kds.setSpillableComplexComponent(sccImpl);
            keyedDataStorage = kds;
        }
        SpillableWindowedKeyedStorage<String, Long> krs = new SpillableWindowedKeyedStorage<>();
        krs.setSpillableComplexComponent(sccImpl);
        keyedRetractionStorage = krs;
        windowedOperator.addComponent("SpillableComplexComponent", sccImpl);
    } else {
        windowStateStorage = new InMemoryWindowedStorage<>();
        if (forSession) {
            keyedDataStorage = new InMemorySessionWindowedStorage<>();
        } else {
            keyedDataStorage = new InMemoryWindowedKeyedStorage<>();
        }
        keyedRetractionStorage = new InMemoryWindowedKeyedStorage<>();
    }
    windowedOperator.setDataStorage(keyedDataStorage);
    windowedOperator.setRetractionStorage(keyedRetractionStorage);
    windowedOperator.setWindowStateStorage(windowStateStorage);
    windowedOperator.setAccumulation(new SumAccumulation());
    return windowedOperator;
}

Example 4 with SpillableWindowedKeyedStorage

use of org.apache.apex.malhar.lib.window.impl.SpillableWindowedKeyedStorage in project apex-malhar by apache.

the class KeyedWindowedOperatorBenchmarkApp method createDataStorage.

@Override
protected WindowedStorage createDataStorage(SpillableComplexComponentImpl sccImpl) {
    if (useInMemoryStorage) {
        return new InMemoryWindowedKeyedStorage();
    }
    SpillableWindowedKeyedStorage dataStorage = new SpillableWindowedKeyedStorage();
    dataStorage.setSpillableComplexComponent(sccImpl);
    return dataStorage;
}