use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast-jet by hazelcast.
the class SlidingWindowPTest method before.
@Before
public void before() {
SlidingWindowPolicy windowDef = slidingWinPolicy(4, 1);
AggregateOperation1<Entry<?, Long>, LongAccumulator, Long> operation = AggregateOperation.withCreate(LongAccumulator::new).andAccumulate((LongAccumulator acc, Entry<?, Long> item) -> acc.add(item.getValue())).andCombine(LongAccumulator::add).andDeduct(hasDeduct ? LongAccumulator::subtract : null).andFinish(LongAccumulator::get);
DistributedFunction<?, Long> keyFn = t -> KEY;
DistributedToLongFunction<Entry<Long, Long>> timestampFn = Entry::getKey;
DistributedSupplier<Processor> procSupplier = singleStageProcessor ? aggregateToSlidingWindowP(singletonList(keyFn), singletonList(timestampFn), TimestampKind.EVENT, windowDef, operation, TimestampedEntry::new) : combineToSlidingWindowP(windowDef, operation, TimestampedEntry::new);
// new supplier to save the last supplied instance
supplier = () -> lastSuppliedProcessor = (SlidingWindowP) procSupplier.get();
}
use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast-jet by hazelcast.
the class SlidingWindowP_twoStageSnapshotTest method before.
@Before
public void before() {
SlidingWindowPolicy windowDef = slidingWinPolicy(4, 1);
AggregateOperation1<Entry<?, Long>, LongAccumulator, Long> aggrOp = AggregateOperation.withCreate(LongAccumulator::new).andAccumulate((LongAccumulator acc, Entry<?, Long> item) -> acc.add(item.getValue())).andCombine(LongAccumulator::add).andDeduct(LongAccumulator::subtract).andFinish(LongAccumulator::get);
DistributedSupplier<Processor> procSupplier1 = Processors.accumulateByFrameP(singletonList((DistributedFunction<? super Entry<Long, Long>, ?>) t -> KEY), singletonList((DistributedToLongFunction<? super Entry<Long, Long>>) Entry::getKey), TimestampKind.EVENT, windowDef, ((AggregateOperation1<? super Entry<Long, Long>, LongAccumulator, ?>) aggrOp).withFinishFn(identity()));
DistributedSupplier<Processor> procSupplier2 = combineToSlidingWindowP(windowDef, aggrOp, TimestampedEntry::new);
// new supplier to save the last supplied instance
stage1Supplier = () -> lastSuppliedStage1Processor = (SlidingWindowP<?, ?, ?, ?>) procSupplier1.get();
stage2Supplier = () -> lastSuppliedStage2Processor = (SlidingWindowP<?, ?, ?, ?>) procSupplier2.get();
}
use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast-jet by hazelcast.
the class WindowGroupTransform method addSlidingWindowTwoStage.
// --------- ---------
// | source0 | ... | sourceN |
// --------- ---------
// | |
// local local
// partitioned partitioned
// v v
// --------------------
// | accumulateByFrameP |
// --------------------
// |
// distributed
// partitioned
// v
// -------------------------
// | combineToSlidingWindowP |
// -------------------------
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDef wDef) {
String namePrefix = p.uniqueVertexName(name(), "-step");
SlidingWindowPolicy winPolicy = wDef.toSlidingWindowPolicy();
Vertex v1 = p.dag.newVertex(namePrefix + '1', accumulateByFrameP(keyFns, nCopies(keyFns.size(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp));
v1.localParallelism(localParallelism());
PlannerVertex pv2 = p.addVertex(this, namePrefix + '2', localParallelism(), combineToSlidingWindowP(winPolicy, aggrOp, mapToOutputFn));
p.addEdges(this, v1, (e, ord) -> e.partitioned(keyFns.get(ord), HASH_CODE));
p.dag.edge(between(v1, pv2.v).distributed().partitioned(entryKey()));
}
use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast-jet-reference-manual by hazelcast.
the class StockExchangeCoreApi method buildDag.
static DAG buildDag() {
// tag::s1[]
DistributedToLongFunction<? super Trade> timestampFn = Trade::timestamp;
DistributedFunction<? super Trade, ?> keyFn = Trade::productId;
SlidingWindowPolicy winPolicy = slidingWinPolicy(SLIDING_WINDOW_LENGTH_MILLIS, SLIDE_STEP_MILLIS);
DAG dag = new DAG();
Vertex tradeSource = dag.newVertex("trade-source", SourceProcessors.<Trade, Long, Trade>streamMapP(TRADES_MAP_NAME, // <1>
alwaysTrue(), // <1>
EventJournalMapEvent::getNewValue, // <2>
JournalInitialPosition.START_FROM_OLDEST, wmGenParams(// <3>
timestampFn, // <4>
limitingLag(SECONDS.toMillis(3)), // <5>
emitByFrame(winPolicy), // <6>
SECONDS.toMillis(3))));
Vertex slidingStage1 = dag.newVertex("sliding-stage-1", Processors.accumulateByFrameP(singletonList(keyFn), singletonList(timestampFn), TimestampKind.EVENT, winPolicy, counting()));
Vertex slidingStage2 = dag.newVertex("sliding-stage-2", Processors.combineToSlidingWindowP(winPolicy, counting(), TimestampedEntry::new));
Vertex formatOutput = dag.newVertex("format-output", mapUsingContextP(ContextFactory.withCreateFn(x -> DateTimeFormatter.ofPattern("HH:mm:ss.SSS")), (DateTimeFormatter timeFormat, TimestampedEntry<String, Long> tse) -> String.format("%s %5s %4d", timeFormat.format(Instant.ofEpochMilli(tse.getTimestamp()).atZone(ZoneId.systemDefault())), tse.getKey(), tse.getValue())));
Vertex sink = dag.newVertex("sink", SinkProcessors.writeFileP(OUTPUT_DIR_NAME));
tradeSource.localParallelism(1);
return dag.edge(between(tradeSource, slidingStage1).partitioned(keyFn, HASH_CODE)).edge(between(slidingStage1, slidingStage2).partitioned(entryKey(), HASH_CODE).distributed()).edge(between(slidingStage2, formatOutput).isolated()).edge(between(formatOutput, sink));
// end::s1[]
}
use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast by hazelcast.
the class SlidingWindow method windowPolicyProvider.
public final FunctionEx<ExpressionEvalContext, SlidingWindowPolicy> windowPolicyProvider() {
QueryParameterMetadata parameterMetadata = ((HazelcastRelOptCluster) getCluster()).getParameterMetadata();
RexToExpressionVisitor visitor = new RexToExpressionVisitor(FAILING_FIELD_TYPE_PROVIDER, parameterMetadata);
if (operator() == HazelcastSqlOperatorTable.TUMBLE) {
Expression<?> windowSizeExpression = operand(2).accept(visitor);
return context -> tumblingWinPolicy(WindowUtils.extractMillis(windowSizeExpression, context));
} else if (operator() == HazelcastSqlOperatorTable.HOP) {
Expression<?> windowSizeExpression = operand(2).accept(visitor);
Expression<?> slideSizeExpression = operand(3).accept(visitor);
return context -> slidingWinPolicy(WindowUtils.extractMillis(windowSizeExpression, context), WindowUtils.extractMillis(slideSizeExpression, context));
} else {
throw new IllegalArgumentException();
}
}
Aggregations