Search in sources :

Example 11 with SlidingWindowPolicy

use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast by hazelcast.

the class WindowGroupTransform method addSlidingWindowTwoStage.

// ---------       ---------
// | source0 | ... | sourceN |
// ---------       ---------
// |               |
// local           local
// partitioned     partitioned
// v               v
// --------------------
// | accumulateByFrameP |
// --------------------
// |
// distributed
// partitioned
// v
// -------------------------
// | combineToSlidingWindowP |
// -------------------------
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDefinition wDef) {
    SlidingWindowPolicy winPolicy = slidingWinPolicy(wDef.windowSize(), wDef.slideBy());
    Vertex v1 = p.dag.newVertex(name() + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateByFrameP(keyFns, nCopies(keyFns.size(), (ToLongFunctionEx<JetEvent<?>>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp));
    v1.localParallelism(determinedLocalParallelism());
    PlannerVertex pv2 = p.addVertex(this, name(), determinedLocalParallelism(), combineToSlidingWindowP(winPolicy, aggrOp, jetEventOfKeyedWindowResultFn()));
    p.addEdges(this, v1, (e, ord) -> e.partitioned(keyFns.get(ord), HASH_CODE));
    p.dag.edge(between(v1, pv2.v).distributed().partitioned(entryKey()));
}
Also used : PlannerVertex(com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex) Vertex(com.hazelcast.jet.core.Vertex) PlannerVertex(com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex) SlidingWindowPolicy(com.hazelcast.jet.core.SlidingWindowPolicy) JetEvent(com.hazelcast.jet.impl.JetEvent)

Example 12 with SlidingWindowPolicy

use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast by hazelcast.

the class CreateDagVisitor method onSlidingWindow.

public Vertex onSlidingWindow(SlidingWindowPhysicalRel rel) {
    int orderingFieldIndex = rel.orderingFieldIndex();
    FunctionEx<ExpressionEvalContext, SlidingWindowPolicy> windowPolicySupplier = rel.windowPolicyProvider();
    // this vertex is used only if there's no aggregation by a window bound
    Vertex vertex = dag.newUniqueVertex("Sliding-Window", flatMapUsingServiceP(ServiceFactories.nonSharedService(ctx -> {
        ExpressionEvalContext evalContext = ExpressionEvalContext.from(ctx);
        SlidingWindowPolicy windowPolicy = windowPolicySupplier.apply(evalContext);
        return row -> WindowUtils.addWindowBounds(row, orderingFieldIndex, windowPolicy);
    }), (BiFunctionEx<Function<JetSqlRow, Traverser<JetSqlRow>>, JetSqlRow, Traverser<JetSqlRow>>) Function::apply));
    connectInput(rel.getInput(), vertex, null);
    return vertex;
}
Also used : ExpressionEvalContext(com.hazelcast.sql.impl.expression.ExpressionEvalContext) Address(com.hazelcast.cluster.Address) Traverser(com.hazelcast.jet.Traverser) ExpressionValues(com.hazelcast.jet.sql.impl.opt.ExpressionValues) PlanObjectKey(com.hazelcast.sql.impl.optimizer.PlanObjectKey) Processors.mapP(com.hazelcast.jet.core.processor.Processors.mapP) SourceProcessors.convenientSourceP(com.hazelcast.jet.core.processor.SourceProcessors.convenientSourceP) QueryParameterMetadata(com.hazelcast.sql.impl.QueryParameterMetadata) Collections.singletonList(java.util.Collections.singletonList) BiFunctionEx(com.hazelcast.function.BiFunctionEx) HazelcastTable(com.hazelcast.jet.sql.impl.schema.HazelcastTable) ObjectArrayKey(com.hazelcast.jet.sql.impl.ObjectArrayKey) SlidingWindowPolicy(com.hazelcast.jet.core.SlidingWindowPolicy) SqlConnectorUtil(com.hazelcast.jet.sql.impl.connector.SqlConnectorUtil) AggregateOperation(com.hazelcast.jet.aggregate.AggregateOperation) Functions.entryKey(com.hazelcast.function.Functions.entryKey) SqlConnectorUtil.getJetSqlConnector(com.hazelcast.jet.sql.impl.connector.SqlConnectorUtil.getJetSqlConnector) DAG(com.hazelcast.jet.core.DAG) RootResultConsumerSink.rootResultConsumerSink(com.hazelcast.jet.sql.impl.processors.RootResultConsumerSink.rootResultConsumerSink) ServiceFactories(com.hazelcast.jet.pipeline.ServiceFactories) FunctionEx(com.hazelcast.function.FunctionEx) KeyedWindowResultFunction(com.hazelcast.jet.core.function.KeyedWindowResultFunction) Predicate(java.util.function.Predicate) Collections.emptyList(java.util.Collections.emptyList) Set(java.util.Set) ConsumerEx(com.hazelcast.function.ConsumerEx) IMapSqlConnector(com.hazelcast.jet.sql.impl.connector.map.IMapSqlConnector) ExpressionUtil(com.hazelcast.jet.sql.impl.ExpressionUtil) Processors.filterUsingServiceP(com.hazelcast.jet.core.processor.Processors.filterUsingServiceP) List(java.util.List) ExpressionEvalContext(com.hazelcast.sql.impl.expression.ExpressionEvalContext) Processors.mapUsingServiceP(com.hazelcast.jet.core.processor.Processors.mapUsingServiceP) Table(com.hazelcast.sql.impl.schema.Table) ComparatorEx(com.hazelcast.function.ComparatorEx) QueryDataType(com.hazelcast.sql.impl.type.QueryDataType) Processors(com.hazelcast.jet.core.processor.Processors) Processors.flatMapUsingServiceP(com.hazelcast.jet.core.processor.Processors.flatMapUsingServiceP) TimestampKind(com.hazelcast.jet.core.TimestampKind) Function(java.util.function.Function) JetSqlRow(com.hazelcast.sql.impl.row.JetSqlRow) HashSet(java.util.HashSet) Edge.from(com.hazelcast.jet.core.Edge.from) Edge(com.hazelcast.jet.core.Edge) Expression(com.hazelcast.sql.impl.expression.Expression) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) VertexWithInputConfig(com.hazelcast.jet.sql.impl.connector.SqlConnector.VertexWithInputConfig) SingleRel(org.apache.calcite.rel.SingleRel) Nullable(javax.annotation.Nullable) JetJoinInfo(com.hazelcast.jet.sql.impl.JetJoinInfo) NodeEngine(com.hazelcast.spi.impl.NodeEngine) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) RelNode(org.apache.calcite.rel.RelNode) Consumer(java.util.function.Consumer) Vertex(com.hazelcast.jet.core.Vertex) ToLongFunctionEx(com.hazelcast.function.ToLongFunctionEx) SqlHashJoinP(com.hazelcast.jet.sql.impl.processors.SqlHashJoinP) ConstantExpression(com.hazelcast.sql.impl.expression.ConstantExpression) Processors.sortP(com.hazelcast.jet.core.processor.Processors.sortP) DefaultSerializationServiceBuilder(com.hazelcast.internal.serialization.impl.DefaultSerializationServiceBuilder) WindowUtils(com.hazelcast.jet.sql.impl.aggregate.WindowUtils) Edge.between(com.hazelcast.jet.core.Edge.between) Vertex(com.hazelcast.jet.core.Vertex) KeyedWindowResultFunction(com.hazelcast.jet.core.function.KeyedWindowResultFunction) Function(java.util.function.Function) SlidingWindowPolicy(com.hazelcast.jet.core.SlidingWindowPolicy) Traverser(com.hazelcast.jet.Traverser) JetSqlRow(com.hazelcast.sql.impl.row.JetSqlRow) BiFunctionEx(com.hazelcast.function.BiFunctionEx)

Example 13 with SlidingWindowPolicy

use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast-jet by hazelcast.

the class WindowAggregateTransform method addSlidingWindowTwoStage.

// --------        ---------
// | source0 | ... | sourceN |
// --------        ---------
// |               |
// local           local
// unicast         unicast
// v               v
// --------------------
// | accumulateByFrameP | keyFn = constantKey()
// --------------------
// |
// distributed
// all-to-one
// v
// -------------------------
// | combineToSlidingWindowP | local parallelism = 1
// -------------------------
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDef wDef) {
    String namePrefix = p.uniqueVertexName(name(), "-step");
    SlidingWindowPolicy winPolicy = wDef.toSlidingWindowPolicy();
    Vertex v1 = p.dag.newVertex(namePrefix + '1', accumulateByFrameP(nCopies(aggrOp.arity(), constantKey()), nCopies(aggrOp.arity(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp));
    v1.localParallelism(localParallelism());
    PlannerVertex pv2 = p.addVertex(this, namePrefix + '2', 1, combineToSlidingWindowP(winPolicy, aggrOp, mapToOutputFn.toKeyedWindowResultFn()));
    p.addEdges(this, v1);
    p.dag.edge(between(v1, pv2.v).distributed().allToOne());
}
Also used : Vertex(com.hazelcast.jet.core.Vertex) PlannerVertex(com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex) PlannerVertex(com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex) SlidingWindowPolicy(com.hazelcast.jet.core.SlidingWindowPolicy) JetEvent(com.hazelcast.jet.impl.pipeline.JetEvent)

Example 14 with SlidingWindowPolicy

use of com.hazelcast.jet.core.SlidingWindowPolicy in project hazelcast-jet by hazelcast.

the class SlidingWindowP_failoverTest method init.

private void init(ProcessingGuarantee guarantee) {
    SlidingWindowPolicy wDef = SlidingWindowPolicy.tumblingWinPolicy(1);
    AggregateOperation1<Object, LongAccumulator, Long> aggrOp = counting();
    p = new SlidingWindowP<>(singletonList(entryKey()), singletonList((DistributedToLongFunction<Entry<?, Long>>) Entry::getValue), wDef, aggrOp, TimestampedEntry::new, true);
    Outbox outbox = new TestOutbox(128);
    Context context = new TestProcessorContext().setProcessingGuarantee(guarantee);
    p.init(outbox, context);
}
Also used : LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) TestProcessorContext(com.hazelcast.jet.core.test.TestProcessorContext) Context(com.hazelcast.jet.core.Processor.Context) Entry(java.util.Map.Entry) TimestampedEntry(com.hazelcast.jet.datamodel.TimestampedEntry) SlidingWindowPolicy(com.hazelcast.jet.core.SlidingWindowPolicy) TestOutbox(com.hazelcast.jet.core.test.TestOutbox) Outbox(com.hazelcast.jet.core.Outbox) TestOutbox(com.hazelcast.jet.core.test.TestOutbox) TestProcessorContext(com.hazelcast.jet.core.test.TestProcessorContext)

Aggregations

SlidingWindowPolicy (com.hazelcast.jet.core.SlidingWindowPolicy)14 Vertex (com.hazelcast.jet.core.Vertex)7 LongAccumulator (com.hazelcast.jet.accumulator.LongAccumulator)6 FunctionEx (com.hazelcast.function.FunctionEx)5 Entry (java.util.Map.Entry)5 ToLongFunctionEx (com.hazelcast.function.ToLongFunctionEx)4 AggregateOperation (com.hazelcast.jet.aggregate.AggregateOperation)4 Processor (com.hazelcast.jet.core.Processor)4 TimestampKind (com.hazelcast.jet.core.TimestampKind)4 Processors.combineToSlidingWindowP (com.hazelcast.jet.core.processor.Processors.combineToSlidingWindowP)4 List (java.util.List)4 Before (org.junit.Before)4 DAG (com.hazelcast.jet.core.DAG)3 SlidingWindowPolicy.slidingWinPolicy (com.hazelcast.jet.core.SlidingWindowPolicy.slidingWinPolicy)3 TimestampedEntry (com.hazelcast.jet.datamodel.TimestampedEntry)3 PlannerVertex (com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex)3 Collections.singletonList (java.util.Collections.singletonList)3 Util.entry (com.hazelcast.jet.Util.entry)2 AggregateOperation1 (com.hazelcast.jet.aggregate.AggregateOperation1)2 Edge (com.hazelcast.jet.core.Edge)2