Examples with SpillableStateStore org.apache.apex.malhar.lib.state.spillable.SpillableStateStore used on opensource projects

Example 1 with SpillableStateStore

use of org.apache.apex.malhar.lib.state.spillable.SpillableStateStore in project apex-malhar by apache.

the class SpillableWindowedKeyedStorage method purge.

@Override
public void purge(long horizonMillis) {
    SpillableStateStore store = scc.getStore();
    if (store instanceof ManagedTimeUnifiedStateImpl) {
        ManagedTimeUnifiedStateImpl timeState = (ManagedTimeUnifiedStateImpl) store;
        long purgeTimeBucket = horizonMillis - timeState.getTimeBucketAssigner().getBucketSpan().getMillis();
        LOG.debug("Purging state less than equal to {}", purgeTimeBucket);
        timeState.purgeTimeBucketsLessThanEqualTo(purgeTimeBucket);
    }
}

Example 2 with SpillableStateStore

use of org.apache.apex.malhar.lib.state.spillable.SpillableStateStore in project apex-malhar by apache.

the class AbstractWindowedOperatorBenchmarkApp method createWindowedOperator.

protected O createWindowedOperator(Configuration conf) {
    SpillableStateStore store = createStore(conf);
    try {
        O windowedOperator = this.windowedOperatorClass.newInstance();
        SpillableComplexComponentImpl sccImpl = new SpillableComplexComponentImpl(store);
        windowedOperator.addComponent("SpillableComplexComponent", sccImpl);
        windowedOperator.setDataStorage(createDataStorage(sccImpl));
        windowedOperator.setRetractionStorage(createRetractionStorage(sccImpl));
        windowedOperator.setWindowStateStorage(new InMemoryWindowedStorage());
        setUpdatedKeyStorage(windowedOperator, conf, sccImpl);
        windowedOperator.setAccumulation(createAccumulation());
        windowedOperator.setAllowedLateness(Duration.millis(ALLOWED_LATENESS));
        windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.standardMinutes(1)));
        // accumulating mode
        windowedOperator.setTriggerOption(TriggerOption.AtWatermark().withEarlyFiringsAtEvery(Duration.standardSeconds(1)).accumulatingFiredPanes().firingOnlyUpdatedPanes());
        windowedOperator.setFixedWatermark(30000);
        return windowedOperator;
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 3 with SpillableStateStore

use of org.apache.apex.malhar.lib.state.spillable.SpillableStateStore in project apex-malhar by apache.

the class SpillableWindowedPlainStorage method purge.

@Override
public void purge(long horizonMillis) {
    SpillableStateStore store = scc.getStore();
    if (store instanceof ManagedTimeUnifiedStateImpl) {
        ManagedTimeUnifiedStateImpl timeState = (ManagedTimeUnifiedStateImpl) store;
        long purgeTimeBucket = horizonMillis - timeState.getTimeBucketAssigner().getBucketSpan().getMillis();
        LOG.debug("Purging state less than equal to {}", purgeTimeBucket);
        timeState.purgeTimeBucketsLessThanEqualTo(purgeTimeBucket);
    }
}