use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast by hazelcast.
the class WindowGroupTransform method addSlidingWindowTwoStage.
// --------- ---------
// | source0 | ... | sourceN |
// --------- ---------
// | |
// local local
// partitioned partitioned
// v v
// --------------------
// | accumulateByFrameP |
// --------------------
// |
// distributed
// partitioned
// v
// -------------------------
// | combineToSlidingWindowP |
// -------------------------
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDefinition wDef) {
SlidingWindowPolicy winPolicy = slidingWinPolicy(wDef.windowSize(), wDef.slideBy());
Vertex v1 = p.dag.newVertex(name() + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateByFrameP(keyFns, nCopies(keyFns.size(), (ToLongFunctionEx<JetEvent<?>>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp));
v1.localParallelism(determinedLocalParallelism());
PlannerVertex pv2 = p.addVertex(this, name(), determinedLocalParallelism(), combineToSlidingWindowP(winPolicy, aggrOp, jetEventOfKeyedWindowResultFn()));
p.addEdges(this, v1, (e, ord) -> e.partitioned(keyFns.get(ord), HASH_CODE));
p.dag.edge(between(v1, pv2.v).distributed().partitioned(entryKey()));
}
use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast by hazelcast.
the class CreateDagVisitor method onSlidingWindow.
public Vertex onSlidingWindow(SlidingWindowPhysicalRel rel) {
int orderingFieldIndex = rel.orderingFieldIndex();
FunctionEx<ExpressionEvalContext, SlidingWindowPolicy> windowPolicySupplier = rel.windowPolicyProvider();
// this vertex is used only if there's no aggregation by a window bound
Vertex vertex = dag.newUniqueVertex("Sliding-Window", flatMapUsingServiceP(ServiceFactories.nonSharedService(ctx -> {
ExpressionEvalContext evalContext = ExpressionEvalContext.from(ctx);
SlidingWindowPolicy windowPolicy = windowPolicySupplier.apply(evalContext);
return row -> WindowUtils.addWindowBounds(row, orderingFieldIndex, windowPolicy);
}), (BiFunctionEx<Function<JetSqlRow, Traverser<JetSqlRow>>, JetSqlRow, Traverser<JetSqlRow>>) Function::apply));
connectInput(rel.getInput(), vertex, null);
return vertex;
}
use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast-jet by hazelcast.
the class WindowAggregateTransform method addSlidingWindowTwoStage.
// -------- ---------
// | source0 | ... | sourceN |
// -------- ---------
// | |
// local local
// unicast unicast
// v v
// --------------------
// | accumulateByFrameP | keyFn = constantKey()
// --------------------
// |
// distributed
// all-to-one
// v
// -------------------------
// | combineToSlidingWindowP | local parallelism = 1
// -------------------------
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDef wDef) {
String namePrefix = p.uniqueVertexName(name(), "-step");
SlidingWindowPolicy winPolicy = wDef.toSlidingWindowPolicy();
Vertex v1 = p.dag.newVertex(namePrefix + '1', accumulateByFrameP(nCopies(aggrOp.arity(), constantKey()), nCopies(aggrOp.arity(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp));
v1.localParallelism(localParallelism());
PlannerVertex pv2 = p.addVertex(this, namePrefix + '2', 1, combineToSlidingWindowP(winPolicy, aggrOp, mapToOutputFn.toKeyedWindowResultFn()));
p.addEdges(this, v1);
p.dag.edge(between(v1, pv2.v).distributed().allToOne());
}
use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast-jet by hazelcast.
the class SlidingWindowP_failoverTest method init.
private void init(ProcessingGuarantee guarantee) {
SlidingWindowPolicy wDef = SlidingWindowPolicy.tumblingWinPolicy(1);
AggregateOperation1<Object, LongAccumulator, Long> aggrOp = counting();
p = new SlidingWindowP<>(singletonList(entryKey()), singletonList((DistributedToLongFunction<Entry<?, Long>>) Entry::getValue), wDef, aggrOp, TimestampedEntry::new, true);
Outbox outbox = new TestOutbox(128);
Context context = new TestProcessorContext().setProcessingGuarantee(guarantee);
p.init(outbox, context);
}
Aggregations