Examples with SpillableWindowedPlainStorage org.apache.apex.malhar.lib.window.impl.SpillableWindowedPlainStorage used on opensource projects

Example 1 with SpillableWindowedPlainStorage

use of org.apache.apex.malhar.lib.window.impl.SpillableWindowedPlainStorage in project apex-malhar by apache.

the class WindowedOperatorBenchmarkApp method createRetractionStorage.

@Override
protected WindowedStorage createRetractionStorage(SpillableComplexComponentImpl sccImpl) {
    SpillableWindowedPlainStorage plainRetractionStorage = new SpillableWindowedPlainStorage();
    plainRetractionStorage.setSpillableComplexComponent(sccImpl);
    return plainRetractionStorage;
}

Example 2 with SpillableWindowedPlainStorage

use of org.apache.apex.malhar.lib.window.impl.SpillableWindowedPlainStorage in project apex-malhar by apache.

the class WindowedOperatorTest method createDefaultWindowedOperator.

private WindowedOperatorImpl<Long, MutableLong, Long> createDefaultWindowedOperator() {
    WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = new WindowedOperatorImpl<>();
    if (useSpillable) {
        sccImpl = new SpillableComplexComponentImpl(testMeta.timeStore);
        // TODO: We don't yet support Spillable data structures for window state storage because SpillableMapImpl does not yet support iterating over all keys.
        windowStateStorage = new InMemoryWindowedStorage<>();
        SpillableWindowedPlainStorage<MutableLong> pds = new SpillableWindowedPlainStorage<>();
        pds.setSpillableComplexComponent(sccImpl);
        plainDataStorage = pds;
        SpillableWindowedPlainStorage<Long> prs = new SpillableWindowedPlainStorage<>();
        prs.setSpillableComplexComponent(sccImpl);
        plainRetractionStorage = prs;
        windowedOperator.addComponent("SpillableComplexComponent", sccImpl);
    } else {
        windowStateStorage = new InMemoryWindowedStorage<>();
        plainDataStorage = new InMemoryWindowedStorage<>();
        plainRetractionStorage = new InMemoryWindowedStorage<>();
    }
    windowedOperator.setDataStorage(plainDataStorage);
    windowedOperator.setRetractionStorage(plainRetractionStorage);
    windowedOperator.setWindowStateStorage(windowStateStorage);
    windowedOperator.setAccumulation(new SumAccumulation());
    return windowedOperator;
}

Example 3 with SpillableWindowedPlainStorage

use of org.apache.apex.malhar.lib.window.impl.SpillableWindowedPlainStorage in project apex-malhar by apache.

the class WindowedOperatorBenchmarkApp method createDataStorage.

@Override
protected WindowedStorage createDataStorage(SpillableComplexComponentImpl sccImpl) {
    SpillableWindowedPlainStorage plainDataStorage = new SpillableWindowedPlainStorage();
    plainDataStorage.setSpillableComplexComponent(sccImpl);
    return plainDataStorage;
}

Example 4 with SpillableWindowedPlainStorage

use of org.apache.apex.malhar.lib.window.impl.SpillableWindowedPlainStorage in project apex-malhar by apache.

the class SpillableWindowedStorageTest method testWindowedPlainStorage.

@Test
public void testWindowedPlainStorage() {
    SpillableComplexComponentImpl sccImpl = new SpillableComplexComponentImpl(testMeta.timeStore);
    SpillableWindowedPlainStorage<Integer> storage = new SpillableWindowedPlainStorage<>();
    Window window1 = new Window.TimeWindow<>(BASETIME + 1000, 10);
    Window window2 = new Window.TimeWindow<>(BASETIME + 1010, 10);
    Window window3 = new Window.TimeWindow<>(BASETIME + 1020, 10);
    storage.setSpillableComplexComponent(sccImpl);
    /*
     * storage.setup() will create Spillable Data Structures
     * storage.getSpillableComplexComponent().setup() will setup these Data Structures.
     * So storage.setup() should be called before storage.getSpillableComplexComponent().setup()
     */
    storage.setup(testMeta.operatorContext);
    storage.getSpillableComplexComponent().setup(testMeta.operatorContext);
    sccImpl.beginWindow(1000);
    storage.put(window1, 1);
    storage.put(window2, 2);
    storage.put(window3, 3);
    sccImpl.endWindow();
    sccImpl.beginWindow(1001);
    storage.put(window1, 4);
    storage.put(window2, 5);
    sccImpl.endWindow();
    sccImpl.beforeCheckpoint(1001);
    SpillableWindowedPlainStorage<Integer> clonedStorage = KryoCloneUtils.cloneObject(storage);
    sccImpl.checkpointed(1001);
    sccImpl.beginWindow(1002);
    storage.put(window1, 6);
    storage.put(window2, 7);
    sccImpl.endWindow();
    Assert.assertEquals(6L, storage.get(window1).longValue());
    Assert.assertEquals(7L, storage.get(window2).longValue());
    Assert.assertEquals(3L, storage.get(window3).longValue());
    sccImpl.beginWindow(1003);
    storage.put(window1, 8);
    storage.put(window2, 9);
    sccImpl.endWindow();
    // simulating crash here
    storage.teardown();
    storage.getSpillableComplexComponent().teardown();
    storage = clonedStorage;
    testMeta.operatorContext.getAttributes().put(Context.OperatorContext.ACTIVATION_WINDOW_ID, 1001L);
    storage.getSpillableComplexComponent().setup(testMeta.operatorContext);
    storage.setup(testMeta.operatorContext);
    // recovery at window 1002
    sccImpl.beginWindow(1002);
    Assert.assertEquals(4L, storage.get(window1).longValue());
    Assert.assertEquals(5L, storage.get(window2).longValue());
    Assert.assertEquals(3L, storage.get(window3).longValue());
}