use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast by hazelcast.
the class CreateDagVisitor method onSlidingWindowAggregate.
public Vertex onSlidingWindowAggregate(SlidingWindowAggregatePhysicalRel rel) {
FunctionEx<JetSqlRow, ?> groupKeyFn = rel.groupKeyFn();
AggregateOperation<?, JetSqlRow> aggregateOperation = rel.aggrOp();
Expression<?> timestampExpression = rel.timestampExpression();
ToLongFunctionEx<JetSqlRow> timestampFn = row -> WindowUtils.extractMillis(timestampExpression.eval(row.getRow(), MOCK_EEC));
SlidingWindowPolicy windowPolicy = rel.windowPolicyProvider().apply(MOCK_EEC);
KeyedWindowResultFunction<? super Object, ? super JetSqlRow, ?> resultMapping = rel.outputValueMapping();
if (rel.numStages() == 1) {
Vertex vertex = dag.newUniqueVertex("Sliding-Window-AggregateByKey", Processors.aggregateToSlidingWindowP(singletonList(groupKeyFn), singletonList(timestampFn), TimestampKind.EVENT, windowPolicy, 0, aggregateOperation, resultMapping));
connectInput(rel.getInput(), vertex, edge -> edge.distributeTo(localMemberAddress).allToOne(""));
return vertex;
} else {
assert rel.numStages() == 2;
Vertex vertex1 = dag.newUniqueVertex("Sliding-Window-AccumulateByKey", Processors.accumulateByFrameP(singletonList(groupKeyFn), singletonList(timestampFn), TimestampKind.EVENT, windowPolicy, aggregateOperation));
Vertex vertex2 = dag.newUniqueVertex("Sliding-Window-CombineByKey", Processors.combineToSlidingWindowP(windowPolicy, aggregateOperation, resultMapping));
connectInput(rel.getInput(), vertex1, edge -> edge.partitioned(groupKeyFn));
dag.edge(between(vertex1, vertex2).distributed().partitioned(entryKey()));
return vertex2;
}
}
use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast by hazelcast.
the class SlidingWindowPTest method before.
@Before
public void before() {
SlidingWindowPolicy winPolicy = slidingWinPolicy(4, 1);
AggregateOperation1<Entry<?, Long>, LongAccumulator, Long> operation = AggregateOperation.withCreate(LongAccumulator::new).andAccumulate((LongAccumulator acc, Entry<?, Long> item) -> acc.add(item.getValue())).andCombine(LongAccumulator::add).andDeduct(hasDeduct ? LongAccumulator::subtract : null).andExportFinish(LongAccumulator::get);
FunctionEx<?, Long> keyFn = t -> KEY;
ToLongFunctionEx<Entry<Long, Long>> timestampFn = Entry::getKey;
SupplierEx<Processor> procSupplier = singleStageProcessor ? aggregateToSlidingWindowP(singletonList(keyFn), singletonList(timestampFn), TimestampKind.EVENT, winPolicy, 0L, operation, KeyedWindowResult::new) : combineToSlidingWindowP(winPolicy, operation, KeyedWindowResult::new);
// new supplier to save the last supplied instance
supplier = () -> lastSuppliedProcessor = (SlidingWindowP) procSupplier.get();
}
use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast by hazelcast.
the class SlidingWindowP_failoverTest method init.
private void init(ProcessingGuarantee guarantee) throws Exception {
SlidingWindowPolicy wDef = SlidingWindowPolicy.tumblingWinPolicy(1);
AggregateOperation1<Object, LongAccumulator, Long> aggrOp = counting();
p = new SlidingWindowP<>(singletonList(entryKey()), singletonList((ToLongFunctionEx<Entry<?, Long>>) Entry::getValue), wDef, 0L, aggrOp, KeyedWindowResult::new, true);
Outbox outbox = new TestOutbox(128);
Context context = new TestProcessorContext().setProcessingGuarantee(guarantee);
p.init(outbox, context);
}
use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast by hazelcast.
the class SlidingWindowP_twoStageSnapshotTest method before.
@Before
public void before() {
SlidingWindowPolicy windowDef = slidingWinPolicy(4, 1);
AggregateOperation1<Entry<?, Long>, LongAccumulator, Long> aggrOp = AggregateOperation.withCreate(LongAccumulator::new).andAccumulate((LongAccumulator acc, Entry<?, Long> item) -> acc.add(item.getValue())).andCombine(LongAccumulator::add).andDeduct(LongAccumulator::subtract).andExportFinish(LongAccumulator::get);
SupplierEx<Processor> procSupplier1 = Processors.accumulateByFrameP(singletonList((FunctionEx<? super Entry<Long, Long>, ?>) t -> KEY), singletonList((ToLongFunctionEx<? super Entry<Long, Long>>) Entry::getKey), TimestampKind.EVENT, windowDef, aggrOp.withIdentityFinish());
SupplierEx<Processor> procSupplier2 = combineToSlidingWindowP(windowDef, aggrOp, KeyedWindowResult::new);
// new supplier to save the last supplied instance
stage1Supplier = () -> lastSuppliedStage1Processor = (SlidingWindowP<?, ?, ?, ?>) procSupplier1.get();
stage2Supplier = () -> lastSuppliedStage2Processor = (SlidingWindowP<?, ?, ?, ?>) procSupplier2.get();
}
use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast by hazelcast.
the class WindowAggregateTransform method addSlidingWindowTwoStage.
// WHEN PRESERVE ORDER IS NOT ACTIVE
// -------- ---------
// | source0 | ... | sourceN |
// -------- ---------
// | |
// local local
// unicast unicast
// v v
// --------------------
// | accumulateByFrameP | keyFn = constantKey()
// --------------------
// |
// distributed
// all-to-one
// v
// -------------------------
// | combineToSlidingWindowP | local parallelism = 1
// -------------------------
// WHEN PRESERVE ORDER IS ACTIVE
// -------- ---------
// | source0 | ... | sourceN |
// -------- ---------
// | |
// isolated isolated
// v v
// --------------------
// | accumulateByFrameP | keyFn = constantKey()
// --------------------
// |
// distributed
// all-to-one
// v
// -------------------------
// | combineToSlidingWindowP | local parallelism = 1
// -------------------------
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDefinition wDef, Context context) {
determineLocalParallelism(LOCAL_PARALLELISM_USE_DEFAULT, context, p.isPreserveOrder());
SlidingWindowPolicy winPolicy = slidingWinPolicy(wDef.windowSize(), wDef.slideBy());
Vertex v1 = p.dag.newVertex(name() + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateByFrameP(nCopies(aggrOp.arity(), new ConstantFunctionEx<>(name().hashCode())), nCopies(aggrOp.arity(), (ToLongFunctionEx<JetEvent<?>>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp));
v1.localParallelism(determinedLocalParallelism());
if (p.isPreserveOrder()) {
p.addEdges(this, v1, Edge::isolated);
} else {
// when preserveOrder is false, we use requested parallelism
// for 1st stage: edge to it is local-unicast, each processor
// can process part of the input which will be combined into
// one result in 2nd stage.
p.addEdges(this, v1);
}
determinedLocalParallelism(1);
PlannerVertex pv2 = p.addVertex(this, name(), determinedLocalParallelism(), combineToSlidingWindowP(winPolicy, aggrOp, jetEventOfWindowResultFn()));
p.dag.edge(between(v1, pv2.v).distributed().allToOne(name().hashCode()));
}
Aggregations