Search in sources :

Example 1 with EventTimePolicy

use of com.hazelcast.jet.core.EventTimePolicy in project hazelcast by hazelcast.

the class JmsSourceBuilder method build.

/**
 * Creates and returns the JMS {@link StreamSource} with the supplied
 * components and the projection function {@code projectionFn}.
 * <p>
 * The given function must be stateless.
 *
 * @param projectionFn the function which creates output object from each
 *                    message
 * @param <T> the type of the items the source emits
 */
@Nonnull
public <T> StreamSource<T> build(@Nonnull FunctionEx<? super Message, ? extends T> projectionFn) {
    String usernameLocal = username;
    String passwordLocal = password;
    String destinationLocal = destinationName;
    ProcessingGuarantee maxGuaranteeLocal = maxGuarantee;
    @SuppressWarnings("UnnecessaryLocalVariable") boolean isTopicLocal = isTopic;
    if (connectionFn == null) {
        connectionFn = factory -> requireNonNull(usernameLocal != null || passwordLocal != null ? factory.createConnection(usernameLocal, passwordLocal) : factory.createConnection());
    }
    if (consumerFn == null) {
        checkNotNull(destinationLocal, "neither consumerFn nor destinationName set");
        consumerFn = session -> session.createConsumer(isTopicLocal ? session.createTopic(destinationLocal) : session.createQueue(destinationLocal));
        if (isTopic) {
            // the user didn't specify a custom consumerFn and we know we're using a non-durable consumer
            // for a topic - there's no point in using any guarantee, see `maxGuarantee`
            maxGuaranteeLocal = ProcessingGuarantee.NONE;
        }
    }
    ProcessingGuarantee maxGuaranteeFinal = maxGuaranteeLocal;
    FunctionEx<? super ConnectionFactory, ? extends Connection> connectionFnLocal = connectionFn;
    @SuppressWarnings("UnnecessaryLocalVariable") SupplierEx<? extends ConnectionFactory> factorySupplierLocal = factorySupplier;
    SupplierEx<? extends Connection> newConnectionFn = () -> connectionFnLocal.apply(factorySupplierLocal.get());
    FunctionEx<? super Session, ? extends MessageConsumer> consumerFnLocal = consumerFn;
    boolean isSharedConsumerLocal = isSharedConsumer;
    FunctionEx<? super Message, ?> messageIdFnLocal = messageIdFn;
    FunctionEx<EventTimePolicy<? super T>, ProcessorMetaSupplier> metaSupplierFactory = policy -> isTopicLocal ? streamJmsTopicP(destinationLocal, isSharedConsumerLocal, maxGuaranteeFinal, policy, newConnectionFn, consumerFnLocal, messageIdFnLocal, projectionFn) : streamJmsQueueP(destinationLocal, maxGuaranteeFinal, policy, newConnectionFn, consumerFnLocal, messageIdFnLocal, projectionFn);
    return Sources.streamFromProcessorWithWatermarks(sourceName(), true, metaSupplierFactory);
}
Also used : ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) FunctionEx(com.hazelcast.function.FunctionEx) Connection(javax.jms.Connection) Util.checkSerializable(com.hazelcast.jet.impl.util.Util.checkSerializable) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) EventTimePolicy(com.hazelcast.jet.core.EventTimePolicy) Preconditions.checkNotNull(com.hazelcast.internal.util.Preconditions.checkNotNull) SupplierEx(com.hazelcast.function.SupplierEx) Session(javax.jms.Session) MessageConsumer(javax.jms.MessageConsumer) SourceProcessors.streamJmsTopicP(com.hazelcast.jet.core.processor.SourceProcessors.streamJmsTopicP) Objects.requireNonNull(java.util.Objects.requireNonNull) SourceProcessors.streamJmsQueueP(com.hazelcast.jet.core.processor.SourceProcessors.streamJmsQueueP) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) Nonnull(javax.annotation.Nonnull) Message(javax.jms.Message) Nullable(javax.annotation.Nullable) ConnectionFactory(javax.jms.ConnectionFactory) EventTimePolicy(com.hazelcast.jet.core.EventTimePolicy) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) Nonnull(javax.annotation.Nonnull)

Example 2 with EventTimePolicy

use of com.hazelcast.jet.core.EventTimePolicy in project hazelcast by hazelcast.

the class TestAbstractSqlConnector method fullScanReader.

@Nonnull
@Override
public Vertex fullScanReader(@Nonnull DAG dag, @Nonnull Table table_, @Nullable Expression<Boolean> predicate, @Nonnull List<Expression<?>> projection, @Nullable FunctionEx<ExpressionEvalContext, EventTimePolicy<JetSqlRow>> eventTimePolicyProvider) {
    TestTable table = (TestTable) table_;
    List<Object[]> rows = table.rows;
    boolean streaming = table.streaming;
    FunctionEx<Context, TestDataGenerator> createContextFn = ctx -> {
        ExpressionEvalContext evalContext = ExpressionEvalContext.from(ctx);
        EventTimePolicy<JetSqlRow> eventTimePolicy = eventTimePolicyProvider == null ? EventTimePolicy.noEventTime() : eventTimePolicyProvider.apply(evalContext);
        return new TestDataGenerator(rows, predicate, projection, evalContext, eventTimePolicy, streaming);
    };
    ProcessorMetaSupplier pms = createProcessorSupplier(createContextFn);
    return dag.newUniqueVertex(table.toString(), pms);
}
Also used : ExpressionEvalContext(com.hazelcast.sql.impl.expression.ExpressionEvalContext) Context(com.hazelcast.jet.core.Processor.Context) Traverser(com.hazelcast.jet.Traverser) Arrays(java.util.Arrays) PlanObjectKey(com.hazelcast.sql.impl.optimizer.PlanObjectKey) EventTimePolicy(com.hazelcast.jet.core.EventTimePolicy) QueryDataTypeFamily(com.hazelcast.sql.impl.type.QueryDataTypeFamily) ArrayList(java.util.ArrayList) JetSqlRow(com.hazelcast.sql.impl.row.JetSqlRow) String.join(java.lang.String.join) Map(java.util.Map) ConstantTableStatistics(com.hazelcast.sql.impl.schema.ConstantTableStatistics) SqlConnector(com.hazelcast.jet.sql.impl.connector.SqlConnector) DAG(com.hazelcast.jet.core.DAG) Expression(com.hazelcast.sql.impl.expression.Expression) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) EventTimeMapper(com.hazelcast.jet.core.EventTimeMapper) QueryException(com.hazelcast.sql.impl.QueryException) FunctionEx(com.hazelcast.function.FunctionEx) SqlService(com.hazelcast.sql.SqlService) NodeEngine(com.hazelcast.spi.impl.NodeEngine) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) JetTable(com.hazelcast.jet.sql.impl.schema.JetTable) Traversers(com.hazelcast.jet.Traversers) ExpressionUtil(com.hazelcast.jet.sql.impl.ExpressionUtil) Collectors.joining(java.util.stream.Collectors.joining) Objects(java.util.Objects) Vertex(com.hazelcast.jet.core.Vertex) TableField(com.hazelcast.sql.impl.schema.TableField) List(java.util.List) ExpressionEvalContext(com.hazelcast.sql.impl.expression.ExpressionEvalContext) QueryDataTypeUtils.resolveTypeForTypeFamily(com.hazelcast.sql.impl.type.QueryDataTypeUtils.resolveTypeForTypeFamily) Context(com.hazelcast.jet.core.Processor.Context) MappingField(com.hazelcast.sql.impl.schema.MappingField) SourceBuilder(com.hazelcast.jet.pipeline.SourceBuilder) Table(com.hazelcast.sql.impl.schema.Table) ExpressionEvalContext(com.hazelcast.sql.impl.expression.ExpressionEvalContext) EventTimePolicy(com.hazelcast.jet.core.EventTimePolicy) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) Nonnull(javax.annotation.Nonnull)

Example 3 with EventTimePolicy

use of com.hazelcast.jet.core.EventTimePolicy in project hazelcast by hazelcast.

the class TestAllTypesSqlConnector method fullScanReader.

@Nonnull
@Override
public Vertex fullScanReader(@Nonnull DAG dag, @Nonnull Table table, @Nullable Expression<Boolean> predicate, @Nonnull List<Expression<?>> projection, @Nullable FunctionEx<ExpressionEvalContext, EventTimePolicy<JetSqlRow>> eventTimePolicyProvider) {
    if (eventTimePolicyProvider != null) {
        throw QueryException.error("Ordering function are not supported for " + TYPE_NAME + " mappings");
    }
    BatchSource<JetSqlRow> source = SourceBuilder.batch("batch", ExpressionEvalContext::from).<JetSqlRow>fillBufferFn((ctx, buf) -> {
        JetSqlRow row = ExpressionUtil.evaluate(predicate, projection, VALUES, ctx);
        if (row != null) {
            buf.add(row);
        }
        buf.close();
    }).build();
    ProcessorMetaSupplier pms = ((BatchSourceTransform<JetSqlRow>) source).metaSupplier;
    return dag.newUniqueVertex(table.toString(), pms);
}
Also used : ExpressionEvalContext(com.hazelcast.sql.impl.expression.ExpressionEvalContext) QueryDataType(com.hazelcast.sql.impl.type.QueryDataType) LocalDateTime(java.time.LocalDateTime) PlanObjectKey(com.hazelcast.sql.impl.optimizer.PlanObjectKey) EventTimePolicy(com.hazelcast.jet.core.EventTimePolicy) JetSqlRow(com.hazelcast.sql.impl.row.JetSqlRow) BigDecimal(java.math.BigDecimal) TEST_SS(com.hazelcast.jet.sql.SqlTestSupport.TEST_SS) Arrays.asList(java.util.Arrays.asList) Map(java.util.Map) ConstantTableStatistics(com.hazelcast.sql.impl.schema.ConstantTableStatistics) LocalTime(java.time.LocalTime) SqlConnector(com.hazelcast.jet.sql.impl.connector.SqlConnector) DAG(com.hazelcast.jet.core.DAG) Expression(com.hazelcast.sql.impl.expression.Expression) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) QueryException(com.hazelcast.sql.impl.QueryException) FunctionEx(com.hazelcast.function.FunctionEx) SqlService(com.hazelcast.sql.SqlService) NodeEngine(com.hazelcast.spi.impl.NodeEngine) BatchSource(com.hazelcast.jet.pipeline.BatchSource) Util.toList(com.hazelcast.jet.impl.util.Util.toList) ImmutableMap(com.google.common.collect.ImmutableMap) BatchSourceTransform(com.hazelcast.jet.impl.pipeline.transform.BatchSourceTransform) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) JetTable(com.hazelcast.jet.sql.impl.schema.JetTable) SqlTestSupport(com.hazelcast.jet.sql.SqlTestSupport) ExpressionUtil(com.hazelcast.jet.sql.impl.ExpressionUtil) Objects(java.util.Objects) Vertex(com.hazelcast.jet.core.Vertex) TableField(com.hazelcast.sql.impl.schema.TableField) List(java.util.List) ExpressionEvalContext(com.hazelcast.sql.impl.expression.ExpressionEvalContext) OffsetDateTime(java.time.OffsetDateTime) LocalDate(java.time.LocalDate) UTC(java.time.ZoneOffset.UTC) MappingField(com.hazelcast.sql.impl.schema.MappingField) SourceBuilder(com.hazelcast.jet.pipeline.SourceBuilder) Table(com.hazelcast.sql.impl.schema.Table) JetSqlRow(com.hazelcast.sql.impl.row.JetSqlRow) BatchSourceTransform(com.hazelcast.jet.impl.pipeline.transform.BatchSourceTransform) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) Nonnull(javax.annotation.Nonnull)

Example 4 with EventTimePolicy

use of com.hazelcast.jet.core.EventTimePolicy in project hazelcast by hazelcast.

the class Planner method createDag.

@SuppressWarnings("rawtypes")
DAG createDag(Context context) {
    pipeline.makeNamesUnique();
    Map<Transform, List<Transform>> adjacencyMap = pipeline.adjacencyMap();
    validateNoLeakage(adjacencyMap);
    checkTopologicalSort(adjacencyMap.entrySet());
    // Find the greatest common denominator of all frame lengths
    // appearing in the pipeline
    long frameSizeGcd = Util.gcd(adjacencyMap.keySet().stream().map(Transform::preferredWatermarkStride).filter(frameSize -> frameSize > 0).mapToLong(i -> i).toArray());
    if (frameSizeGcd == 0) {
        // even if there are no window aggregations, we want the watermarks for latency debugging
        frameSizeGcd = MAXIMUM_WATERMARK_GAP;
    }
    if (frameSizeGcd > MAXIMUM_WATERMARK_GAP) {
        frameSizeGcd = Util.gcd(frameSizeGcd, MAXIMUM_WATERMARK_GAP);
    }
    LoggingUtil.logFine(LOGGER, "Watermarks in the pipeline will be throttled to %d", frameSizeGcd);
    // Update watermark throttling frame length on all transforms with the determined length
    for (Transform transform : adjacencyMap.keySet()) {
        if (transform instanceof StreamSourceTransform) {
            StreamSourceTransform t = (StreamSourceTransform) transform;
            EventTimePolicy policy = t.getEventTimePolicy();
            if (policy != null) {
                t.setEventTimePolicy(withFrameSize(policy, frameSizeGcd));
            }
        } else if (transform instanceof TimestampTransform) {
            TimestampTransform t = (TimestampTransform) transform;
            t.setEventTimePolicy(withFrameSize(t.getEventTimePolicy(), frameSizeGcd));
        }
    }
    // fuse subsequent map/filter/flatMap transforms into one
    Map<Transform, List<Transform>> originalParents = new HashMap<>();
    List<Transform> transforms = new ArrayList<>(adjacencyMap.keySet());
    for (int i = 0; i < transforms.size(); i++) {
        Transform transform = transforms.get(i);
        List<Transform> chain = findFusableChain(transform, adjacencyMap);
        if (chain == null) {
            continue;
        }
        // remove transforms in the chain and replace the parent with a fused transform
        transforms.removeAll(chain.subList(1, chain.size()));
        Transform fused = fuseFlatMapTransforms(chain);
        transforms.set(i, fused);
        Transform lastInChain = chain.get(chain.size() - 1);
        for (Transform downstream : adjacencyMap.get(lastInChain)) {
            originalParents.put(downstream, new ArrayList<>(downstream.upstream()));
            downstream.upstream().replaceAll(p -> p == lastInChain ? fused : p);
        }
    }
    for (Transform transform : transforms) {
        transform.addToDag(this, context);
    }
    // restore original parents
    for (Entry<Transform, List<Transform>> en : originalParents.entrySet()) {
        List<Transform> upstream = en.getKey().upstream();
        for (int i = 0; i < upstream.size(); i++) {
            en.getKey().upstream().set(i, en.getValue().get(i));
        }
    }
    return dag;
}
Also used : Traverser(com.hazelcast.jet.Traverser) LoggingUtil(com.hazelcast.jet.impl.util.LoggingUtil) Processor(com.hazelcast.jet.core.Processor) HashMap(java.util.HashMap) ObjIntConsumer(java.util.function.ObjIntConsumer) EventTimePolicy(com.hazelcast.jet.core.EventTimePolicy) ArrayList(java.util.ArrayList) SinkTransform(com.hazelcast.jet.impl.pipeline.transform.SinkTransform) TopologicalSorter.checkTopologicalSort(com.hazelcast.jet.impl.TopologicalSorter.checkTopologicalSort) ILogger(com.hazelcast.logging.ILogger) Map(java.util.Map) MapTransform(com.hazelcast.jet.impl.pipeline.transform.MapTransform) TimestampTransform(com.hazelcast.jet.impl.pipeline.transform.TimestampTransform) Edge.from(com.hazelcast.jet.core.Edge.from) DAG(com.hazelcast.jet.core.DAG) Edge(com.hazelcast.jet.core.Edge) Nonnull(javax.annotation.Nonnull) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) FunctionEx(com.hazelcast.function.FunctionEx) Logger(com.hazelcast.logging.Logger) Util.toList(com.hazelcast.jet.impl.util.Util.toList) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) Traversers(com.hazelcast.jet.Traversers) Collectors(java.util.stream.Collectors) SupplierEx(com.hazelcast.function.SupplierEx) Transform(com.hazelcast.jet.impl.pipeline.transform.Transform) Consumer(java.util.function.Consumer) Vertex(com.hazelcast.jet.core.Vertex) Util(com.hazelcast.jet.impl.util.Util) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) Entry(java.util.Map.Entry) StreamSourceTransform(com.hazelcast.jet.impl.pipeline.transform.StreamSourceTransform) EventTimePolicy.eventTimePolicy(com.hazelcast.jet.core.EventTimePolicy.eventTimePolicy) FlatMapTransform(com.hazelcast.jet.impl.pipeline.transform.FlatMapTransform) Context(com.hazelcast.jet.impl.pipeline.PipelineImpl.Context) EventTimePolicy(com.hazelcast.jet.core.EventTimePolicy) HashMap(java.util.HashMap) StreamSourceTransform(com.hazelcast.jet.impl.pipeline.transform.StreamSourceTransform) ArrayList(java.util.ArrayList) TimestampTransform(com.hazelcast.jet.impl.pipeline.transform.TimestampTransform) ArrayList(java.util.ArrayList) Util.toList(com.hazelcast.jet.impl.util.Util.toList) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) SinkTransform(com.hazelcast.jet.impl.pipeline.transform.SinkTransform) MapTransform(com.hazelcast.jet.impl.pipeline.transform.MapTransform) TimestampTransform(com.hazelcast.jet.impl.pipeline.transform.TimestampTransform) Transform(com.hazelcast.jet.impl.pipeline.transform.Transform) StreamSourceTransform(com.hazelcast.jet.impl.pipeline.transform.StreamSourceTransform) FlatMapTransform(com.hazelcast.jet.impl.pipeline.transform.FlatMapTransform)

Aggregations

FunctionEx (com.hazelcast.function.FunctionEx)4 EventTimePolicy (com.hazelcast.jet.core.EventTimePolicy)4 ProcessorMetaSupplier (com.hazelcast.jet.core.ProcessorMetaSupplier)4 Nonnull (javax.annotation.Nonnull)4 DAG (com.hazelcast.jet.core.DAG)3 Vertex (com.hazelcast.jet.core.Vertex)3 List (java.util.List)3 Map (java.util.Map)3 Nullable (javax.annotation.Nullable)3 SupplierEx (com.hazelcast.function.SupplierEx)2 Traverser (com.hazelcast.jet.Traverser)2 Traversers (com.hazelcast.jet.Traversers)2 Util.toList (com.hazelcast.jet.impl.util.Util.toList)2 SourceBuilder (com.hazelcast.jet.pipeline.SourceBuilder)2 ExpressionUtil (com.hazelcast.jet.sql.impl.ExpressionUtil)2 SqlConnector (com.hazelcast.jet.sql.impl.connector.SqlConnector)2 JetTable (com.hazelcast.jet.sql.impl.schema.JetTable)2 NodeEngine (com.hazelcast.spi.impl.NodeEngine)2 SqlService (com.hazelcast.sql.SqlService)2 QueryException (com.hazelcast.sql.impl.QueryException)2