Examples with SpillableComplexComponentImpl org.apache.apex.malhar.lib.state.spillable.SpillableComplexComponentImpl used on opensource projects

Example 1 with SpillableComplexComponentImpl

use of org.apache.apex.malhar.lib.state.spillable.SpillableComplexComponentImpl in project apex-malhar by apache.

the class AbstractWindowedOperatorBenchmarkApp method createWindowedOperator.

protected O createWindowedOperator(Configuration conf) {
    SpillableStateStore store = createStore(conf);
    try {
        O windowedOperator = this.windowedOperatorClass.newInstance();
        SpillableComplexComponentImpl sccImpl = new SpillableComplexComponentImpl(store);
        windowedOperator.addComponent("SpillableComplexComponent", sccImpl);
        windowedOperator.setDataStorage(createDataStorage(sccImpl));
        windowedOperator.setRetractionStorage(createRetractionStorage(sccImpl));
        windowedOperator.setWindowStateStorage(new InMemoryWindowedStorage());
        setUpdatedKeyStorage(windowedOperator, conf, sccImpl);
        windowedOperator.setAccumulation(createAccumulation());
        windowedOperator.setAllowedLateness(Duration.millis(ALLOWED_LATENESS));
        windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.standardMinutes(1)));
        // accumulating mode
        windowedOperator.setTriggerOption(TriggerOption.AtWatermark().withEarlyFiringsAtEvery(Duration.standardSeconds(1)).accumulatingFiredPanes().firingOnlyUpdatedPanes());
        windowedOperator.setFixedWatermark(30000);
        return windowedOperator;
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 2 with SpillableComplexComponentImpl

use of org.apache.apex.malhar.lib.state.spillable.SpillableComplexComponentImpl in project apex-malhar by apache.

the class WindowedOperatorTest method createDefaultWindowedOperator.

private WindowedOperatorImpl<Long, MutableLong, Long> createDefaultWindowedOperator() {
    WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = new WindowedOperatorImpl<>();
    if (useSpillable) {
        sccImpl = new SpillableComplexComponentImpl(testMeta.timeStore);
        // TODO: We don't yet support Spillable data structures for window state storage because SpillableMapImpl does not yet support iterating over all keys.
        windowStateStorage = new InMemoryWindowedStorage<>();
        SpillableWindowedPlainStorage<MutableLong> pds = new SpillableWindowedPlainStorage<>();
        pds.setSpillableComplexComponent(sccImpl);
        plainDataStorage = pds;
        SpillableWindowedPlainStorage<Long> prs = new SpillableWindowedPlainStorage<>();
        prs.setSpillableComplexComponent(sccImpl);
        plainRetractionStorage = prs;
        windowedOperator.addComponent("SpillableComplexComponent", sccImpl);
    } else {
        windowStateStorage = new InMemoryWindowedStorage<>();
        plainDataStorage = new InMemoryWindowedStorage<>();
        plainRetractionStorage = new InMemoryWindowedStorage<>();
    }
    windowedOperator.setDataStorage(plainDataStorage);
    windowedOperator.setRetractionStorage(plainRetractionStorage);
    windowedOperator.setWindowStateStorage(windowStateStorage);
    windowedOperator.setAccumulation(new SumAccumulation());
    return windowedOperator;
}

Example 3 with SpillableComplexComponentImpl

use of org.apache.apex.malhar.lib.state.spillable.SpillableComplexComponentImpl in project apex-malhar by apache.

the class SpillableWindowedStorageTest method testWindowedPlainStorage.

@Test
public void testWindowedPlainStorage() {
    SpillableComplexComponentImpl sccImpl = new SpillableComplexComponentImpl(testMeta.timeStore);
    SpillableWindowedPlainStorage<Integer> storage = new SpillableWindowedPlainStorage<>();
    Window window1 = new Window.TimeWindow<>(BASETIME + 1000, 10);
    Window window2 = new Window.TimeWindow<>(BASETIME + 1010, 10);
    Window window3 = new Window.TimeWindow<>(BASETIME + 1020, 10);
    storage.setSpillableComplexComponent(sccImpl);
    /*
     * storage.setup() will create Spillable Data Structures
     * storage.getSpillableComplexComponent().setup() will setup these Data Structures.
     * So storage.setup() should be called before storage.getSpillableComplexComponent().setup()
     */
    storage.setup(testMeta.operatorContext);
    storage.getSpillableComplexComponent().setup(testMeta.operatorContext);
    sccImpl.beginWindow(1000);
    storage.put(window1, 1);
    storage.put(window2, 2);
    storage.put(window3, 3);
    sccImpl.endWindow();
    sccImpl.beginWindow(1001);
    storage.put(window1, 4);
    storage.put(window2, 5);
    sccImpl.endWindow();
    sccImpl.beforeCheckpoint(1001);
    SpillableWindowedPlainStorage<Integer> clonedStorage = KryoCloneUtils.cloneObject(storage);
    sccImpl.checkpointed(1001);
    sccImpl.beginWindow(1002);
    storage.put(window1, 6);
    storage.put(window2, 7);
    sccImpl.endWindow();
    Assert.assertEquals(6L, storage.get(window1).longValue());
    Assert.assertEquals(7L, storage.get(window2).longValue());
    Assert.assertEquals(3L, storage.get(window3).longValue());
    sccImpl.beginWindow(1003);
    storage.put(window1, 8);
    storage.put(window2, 9);
    sccImpl.endWindow();
    // simulating crash here
    storage.teardown();
    storage.getSpillableComplexComponent().teardown();
    storage = clonedStorage;
    testMeta.operatorContext.getAttributes().put(Context.OperatorContext.ACTIVATION_WINDOW_ID, 1001L);
    storage.getSpillableComplexComponent().setup(testMeta.operatorContext);
    storage.setup(testMeta.operatorContext);
    // recovery at window 1002
    sccImpl.beginWindow(1002);
    Assert.assertEquals(4L, storage.get(window1).longValue());
    Assert.assertEquals(5L, storage.get(window2).longValue());
    Assert.assertEquals(3L, storage.get(window3).longValue());
}

Example 4 with SpillableComplexComponentImpl

use of org.apache.apex.malhar.lib.state.spillable.SpillableComplexComponentImpl in project apex-malhar by apache.

the class SpillableWindowedStorageTest method testWindowedKeyedStorage.

@Test
public void testWindowedKeyedStorage() {
    SpillableComplexComponentImpl sccImpl = new SpillableComplexComponentImpl(testMeta.timeStore);
    SpillableWindowedKeyedStorage<String, Integer> storage = new SpillableWindowedKeyedStorage<>();
    Window window1 = new Window.TimeWindow<>(BASETIME + 1000, 10);
    Window window2 = new Window.TimeWindow<>(BASETIME + 1010, 10);
    Window window3 = new Window.TimeWindow<>(BASETIME + 1020, 10);
    storage.setSpillableComplexComponent(sccImpl);
    /*
     * storage.setup() will create Spillable Data Structures
     * storage.getSpillableComplexComponent().setup() will setup these Data Structures.
     * So storage.setup() should be called before storage.getSpillableComplexComponent().setup()
     */
    storage.setup(testMeta.operatorContext);
    storage.getSpillableComplexComponent().setup(testMeta.operatorContext);
    sccImpl.beginWindow(1000);
    storage.put(window1, "x", 1);
    storage.put(window2, "x", 2);
    storage.put(window3, "x", 3);
    sccImpl.endWindow();
    sccImpl.beginWindow(1001);
    storage.put(window1, "x", 4);
    storage.put(window2, "x", 5);
    sccImpl.endWindow();
    sccImpl.beforeCheckpoint(1001);
    SpillableWindowedKeyedStorage<String, Integer> clonedStorage = KryoCloneUtils.cloneObject(storage);
    sccImpl.checkpointed(1001);
    sccImpl.beginWindow(1002);
    storage.put(window1, "x", 6);
    storage.put(window2, "x", 7);
    storage.put(window2, "y", 8);
    sccImpl.endWindow();
    Assert.assertEquals(6L, storage.get(window1, "x").longValue());
    Assert.assertEquals(7L, storage.get(window2, "x").longValue());
    Assert.assertEquals(3L, storage.get(window3, "x").longValue());
    Assert.assertEquals(8L, storage.get(window2, "y").longValue());
    // simulating crash here
    storage.teardown();
    storage.getSpillableComplexComponent().teardown();
    storage = clonedStorage;
    testMeta.operatorContext.getAttributes().put(Context.OperatorContext.ACTIVATION_WINDOW_ID, 1001L);
    storage.getSpillableComplexComponent().setup(testMeta.operatorContext);
    storage.setup(testMeta.operatorContext);
    // recovery at window 1002
    sccImpl.beginWindow(1002);
    Assert.assertEquals(4L, storage.get(window1, "x").longValue());
    Assert.assertEquals(5L, storage.get(window2, "x").longValue());
    Assert.assertEquals(3L, storage.get(window3, "x").longValue());
    Assert.assertNull(storage.get(window2, "y"));
}

Example 5 with SpillableComplexComponentImpl

use of org.apache.apex.malhar.lib.state.spillable.SpillableComplexComponentImpl in project apex-malhar by apache.

the class WindowedOperatorTest method createDefaultKeyedWindowedOperator.

private KeyedWindowedOperatorImpl<String, Long, MutableLong, Long> createDefaultKeyedWindowedOperator(boolean forSession) {
    KeyedWindowedOperatorImpl<String, Long, MutableLong, Long> windowedOperator = new KeyedWindowedOperatorImpl<>();
    if (useSpillable) {
        sccImpl = new SpillableComplexComponentImpl(testMeta.timeStore);
        // TODO: We don't yet support Spillable data structures for window state storage because SpillableMapImpl does not yet support iterating over all keys.
        windowStateStorage = new InMemoryWindowedStorage<>();
        if (forSession) {
            SpillableSessionWindowedStorage<String, MutableLong> sws = new SpillableSessionWindowedStorage<>();
            sws.setSpillableComplexComponent(sccImpl);
            keyedDataStorage = sws;
        } else {
            SpillableWindowedKeyedStorage<String, MutableLong> kds = new SpillableWindowedKeyedStorage<>();
            kds.setSpillableComplexComponent(sccImpl);
            keyedDataStorage = kds;
        }
        SpillableWindowedKeyedStorage<String, Long> krs = new SpillableWindowedKeyedStorage<>();
        krs.setSpillableComplexComponent(sccImpl);
        keyedRetractionStorage = krs;
        windowedOperator.addComponent("SpillableComplexComponent", sccImpl);
    } else {
        windowStateStorage = new InMemoryWindowedStorage<>();
        if (forSession) {
            keyedDataStorage = new InMemorySessionWindowedStorage<>();
        } else {
            keyedDataStorage = new InMemoryWindowedKeyedStorage<>();
        }
        keyedRetractionStorage = new InMemoryWindowedKeyedStorage<>();
    }
    windowedOperator.setDataStorage(keyedDataStorage);
    windowedOperator.setRetractionStorage(keyedRetractionStorage);
    windowedOperator.setWindowStateStorage(windowStateStorage);
    windowedOperator.setAccumulation(new SumAccumulation());
    return windowedOperator;
}