Search in sources :

Example 6 with ProcessorMetaSupplier

use of com.hazelcast.jet.core.ProcessorMetaSupplier in project hazelcast by hazelcast.

the class PartitionedProcessorTransform method flatMapUsingServiceAsyncBatchedPartitionedTransform.

public static <S, T, K, R> PartitionedProcessorTransform<T, K> flatMapUsingServiceAsyncBatchedPartitionedTransform(@Nonnull Transform upstream, @Nonnull String operationName, @Nonnull ServiceFactory<?, S> serviceFactory, int maxConcurrentOps, int maxBatchSize, @Nonnull BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<Traverser<R>>> flatMapAsyncFn, @Nonnull FunctionEx<? super T, ? extends K> partitionKeyFn) {
    String name = operationName + "UsingPartitionedServiceAsync";
    ProcessorSupplier supplier = flatMapUsingServiceAsyncBatchedP(serviceFactory, maxConcurrentOps, maxBatchSize, flatMapAsyncFn);
    ProcessorMetaSupplier metaSupplier = ProcessorMetaSupplier.of(getPreferredLP(serviceFactory), serviceFactory.permission(), supplier);
    return new PartitionedProcessorTransform<>(name, upstream, metaSupplier, partitionKeyFn);
}
Also used : ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier)

Example 7 with ProcessorMetaSupplier

use of com.hazelcast.jet.core.ProcessorMetaSupplier in project hazelcast by hazelcast.

the class ProcessorTransform method flatMapUsingServiceAsyncBatchedTransform.

public static <S, T, R> ProcessorTransform flatMapUsingServiceAsyncBatchedTransform(@Nonnull Transform upstream, @Nonnull String operationName, @Nonnull ServiceFactory<?, S> serviceFactory, int maxConcurrentOps, int maxBatchSize, @Nonnull BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<Traverser<R>>> flatMapAsyncFn) {
    String name = operationName + "UsingServiceAsyncBatched";
    ProcessorSupplier supplier = flatMapUsingServiceAsyncBatchedP(serviceFactory, maxConcurrentOps, maxBatchSize, flatMapAsyncFn);
    ProcessorMetaSupplier metaSupplier = ProcessorMetaSupplier.of(getPreferredLP(serviceFactory), serviceFactory.permission(), supplier);
    return new ProcessorTransform(name, upstream, metaSupplier);
}
Also used : ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier)

Example 8 with ProcessorMetaSupplier

use of com.hazelcast.jet.core.ProcessorMetaSupplier in project hazelcast by hazelcast.

the class StreamSourceTransform method addToDag.

@Override
public void addToDag(Planner p, Context context) {
    if (emitsWatermarks || eventTimePolicy == null) {
        // Reached when the source either emits both JetEvents and watermarks
        // or neither. In these cases we don't have to insert watermarks.
        final ProcessorMetaSupplier metaSupplier = metaSupplierFn.apply(eventTimePolicy != null ? eventTimePolicy : noEventTime());
        determineLocalParallelism(metaSupplier.preferredLocalParallelism(), context, false);
        p.addVertex(this, name(), determinedLocalParallelism(), metaSupplier);
    } else {
        // ------------
        // |  sourceP   |
        // ------------
        // |
        // isolated
        // v
        // -------------
        // |  insertWmP  |
        // -------------
        String v1name = name();
        final ProcessorMetaSupplier metaSupplier = metaSupplierFn.apply(eventTimePolicy);
        determineLocalParallelism(metaSupplier.preferredLocalParallelism(), context, false);
        Vertex v1 = p.dag.newVertex(v1name, metaSupplier).localParallelism(determinedLocalParallelism());
        PlannerVertex pv2 = p.addVertex(this, v1name + "-add-timestamps", determinedLocalParallelism(), insertWatermarksP(eventTimePolicy));
        p.dag.edge(between(v1, pv2.v).isolated());
    }
}
Also used : Vertex(com.hazelcast.jet.core.Vertex) PlannerVertex(com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex) PlannerVertex(com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier)

Example 9 with ProcessorMetaSupplier

use of com.hazelcast.jet.core.ProcessorMetaSupplier in project hazelcast by hazelcast.

the class TestAbstractSqlConnector method fullScanReader.

@Nonnull
@Override
public Vertex fullScanReader(@Nonnull DAG dag, @Nonnull Table table_, @Nullable Expression<Boolean> predicate, @Nonnull List<Expression<?>> projection, @Nullable FunctionEx<ExpressionEvalContext, EventTimePolicy<JetSqlRow>> eventTimePolicyProvider) {
    TestTable table = (TestTable) table_;
    List<Object[]> rows = table.rows;
    boolean streaming = table.streaming;
    FunctionEx<Context, TestDataGenerator> createContextFn = ctx -> {
        ExpressionEvalContext evalContext = ExpressionEvalContext.from(ctx);
        EventTimePolicy<JetSqlRow> eventTimePolicy = eventTimePolicyProvider == null ? EventTimePolicy.noEventTime() : eventTimePolicyProvider.apply(evalContext);
        return new TestDataGenerator(rows, predicate, projection, evalContext, eventTimePolicy, streaming);
    };
    ProcessorMetaSupplier pms = createProcessorSupplier(createContextFn);
    return dag.newUniqueVertex(table.toString(), pms);
}
Also used : ExpressionEvalContext(com.hazelcast.sql.impl.expression.ExpressionEvalContext) Context(com.hazelcast.jet.core.Processor.Context) Traverser(com.hazelcast.jet.Traverser) Arrays(java.util.Arrays) PlanObjectKey(com.hazelcast.sql.impl.optimizer.PlanObjectKey) EventTimePolicy(com.hazelcast.jet.core.EventTimePolicy) QueryDataTypeFamily(com.hazelcast.sql.impl.type.QueryDataTypeFamily) ArrayList(java.util.ArrayList) JetSqlRow(com.hazelcast.sql.impl.row.JetSqlRow) String.join(java.lang.String.join) Map(java.util.Map) ConstantTableStatistics(com.hazelcast.sql.impl.schema.ConstantTableStatistics) SqlConnector(com.hazelcast.jet.sql.impl.connector.SqlConnector) DAG(com.hazelcast.jet.core.DAG) Expression(com.hazelcast.sql.impl.expression.Expression) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) EventTimeMapper(com.hazelcast.jet.core.EventTimeMapper) QueryException(com.hazelcast.sql.impl.QueryException) FunctionEx(com.hazelcast.function.FunctionEx) SqlService(com.hazelcast.sql.SqlService) NodeEngine(com.hazelcast.spi.impl.NodeEngine) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) JetTable(com.hazelcast.jet.sql.impl.schema.JetTable) Traversers(com.hazelcast.jet.Traversers) ExpressionUtil(com.hazelcast.jet.sql.impl.ExpressionUtil) Collectors.joining(java.util.stream.Collectors.joining) Objects(java.util.Objects) Vertex(com.hazelcast.jet.core.Vertex) TableField(com.hazelcast.sql.impl.schema.TableField) List(java.util.List) ExpressionEvalContext(com.hazelcast.sql.impl.expression.ExpressionEvalContext) QueryDataTypeUtils.resolveTypeForTypeFamily(com.hazelcast.sql.impl.type.QueryDataTypeUtils.resolveTypeForTypeFamily) Context(com.hazelcast.jet.core.Processor.Context) MappingField(com.hazelcast.sql.impl.schema.MappingField) SourceBuilder(com.hazelcast.jet.pipeline.SourceBuilder) Table(com.hazelcast.sql.impl.schema.Table) ExpressionEvalContext(com.hazelcast.sql.impl.expression.ExpressionEvalContext) EventTimePolicy(com.hazelcast.jet.core.EventTimePolicy) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) Nonnull(javax.annotation.Nonnull)

Example 10 with ProcessorMetaSupplier

use of com.hazelcast.jet.core.ProcessorMetaSupplier in project hazelcast by hazelcast.

the class TestAllTypesSqlConnector method fullScanReader.

@Nonnull
@Override
public Vertex fullScanReader(@Nonnull DAG dag, @Nonnull Table table, @Nullable Expression<Boolean> predicate, @Nonnull List<Expression<?>> projection, @Nullable FunctionEx<ExpressionEvalContext, EventTimePolicy<JetSqlRow>> eventTimePolicyProvider) {
    if (eventTimePolicyProvider != null) {
        throw QueryException.error("Ordering function are not supported for " + TYPE_NAME + " mappings");
    }
    BatchSource<JetSqlRow> source = SourceBuilder.batch("batch", ExpressionEvalContext::from).<JetSqlRow>fillBufferFn((ctx, buf) -> {
        JetSqlRow row = ExpressionUtil.evaluate(predicate, projection, VALUES, ctx);
        if (row != null) {
            buf.add(row);
        }
        buf.close();
    }).build();
    ProcessorMetaSupplier pms = ((BatchSourceTransform<JetSqlRow>) source).metaSupplier;
    return dag.newUniqueVertex(table.toString(), pms);
}
Also used : ExpressionEvalContext(com.hazelcast.sql.impl.expression.ExpressionEvalContext) QueryDataType(com.hazelcast.sql.impl.type.QueryDataType) LocalDateTime(java.time.LocalDateTime) PlanObjectKey(com.hazelcast.sql.impl.optimizer.PlanObjectKey) EventTimePolicy(com.hazelcast.jet.core.EventTimePolicy) JetSqlRow(com.hazelcast.sql.impl.row.JetSqlRow) BigDecimal(java.math.BigDecimal) TEST_SS(com.hazelcast.jet.sql.SqlTestSupport.TEST_SS) Arrays.asList(java.util.Arrays.asList) Map(java.util.Map) ConstantTableStatistics(com.hazelcast.sql.impl.schema.ConstantTableStatistics) LocalTime(java.time.LocalTime) SqlConnector(com.hazelcast.jet.sql.impl.connector.SqlConnector) DAG(com.hazelcast.jet.core.DAG) Expression(com.hazelcast.sql.impl.expression.Expression) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) QueryException(com.hazelcast.sql.impl.QueryException) FunctionEx(com.hazelcast.function.FunctionEx) SqlService(com.hazelcast.sql.SqlService) NodeEngine(com.hazelcast.spi.impl.NodeEngine) BatchSource(com.hazelcast.jet.pipeline.BatchSource) Util.toList(com.hazelcast.jet.impl.util.Util.toList) ImmutableMap(com.google.common.collect.ImmutableMap) BatchSourceTransform(com.hazelcast.jet.impl.pipeline.transform.BatchSourceTransform) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) JetTable(com.hazelcast.jet.sql.impl.schema.JetTable) SqlTestSupport(com.hazelcast.jet.sql.SqlTestSupport) ExpressionUtil(com.hazelcast.jet.sql.impl.ExpressionUtil) Objects(java.util.Objects) Vertex(com.hazelcast.jet.core.Vertex) TableField(com.hazelcast.sql.impl.schema.TableField) List(java.util.List) ExpressionEvalContext(com.hazelcast.sql.impl.expression.ExpressionEvalContext) OffsetDateTime(java.time.OffsetDateTime) LocalDate(java.time.LocalDate) UTC(java.time.ZoneOffset.UTC) MappingField(com.hazelcast.sql.impl.schema.MappingField) SourceBuilder(com.hazelcast.jet.pipeline.SourceBuilder) Table(com.hazelcast.sql.impl.schema.Table) JetSqlRow(com.hazelcast.sql.impl.row.JetSqlRow) BatchSourceTransform(com.hazelcast.jet.impl.pipeline.transform.BatchSourceTransform) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) Nonnull(javax.annotation.Nonnull)

Aggregations

ProcessorMetaSupplier (com.hazelcast.jet.core.ProcessorMetaSupplier)21 ProcessorSupplier (com.hazelcast.jet.core.ProcessorSupplier)11 Nonnull (javax.annotation.Nonnull)8 Test (org.junit.Test)6 Vertex (com.hazelcast.jet.core.Vertex)5 QuickTest (com.hazelcast.test.annotation.QuickTest)5 List (java.util.List)5 Address (com.hazelcast.cluster.Address)4 FunctionEx (com.hazelcast.function.FunctionEx)4 DAG (com.hazelcast.jet.core.DAG)4 TestProcessorContext (com.hazelcast.jet.core.test.TestProcessorContext)4 JetSqlRow (com.hazelcast.sql.impl.row.JetSqlRow)4 ParallelJVMTest (com.hazelcast.test.annotation.ParallelJVMTest)4 Map (java.util.Map)4 EventTimePolicy (com.hazelcast.jet.core.EventTimePolicy)3 Function (java.util.function.Function)3 Nullable (javax.annotation.Nullable)3 HazelcastInstance (com.hazelcast.core.HazelcastInstance)2 MemberInfo (com.hazelcast.internal.cluster.MemberInfo)2 EdgeConfig (com.hazelcast.jet.config.EdgeConfig)2