Examples with StreamSourceTransform com.hazelcast.jet.impl.pipeline.transform.StreamSourceTransform used on opensource projects

Example 1 with StreamSourceTransform

use of com.hazelcast.jet.impl.pipeline.transform.StreamSourceTransform in project hazelcast-jet by hazelcast.

the class Planner method createDag.

DAG createDag() {
    Map<Transform, List<Transform>> adjacencyMap = pipeline.adjacencyMap();
    validateNoLeakage(adjacencyMap);
    // Calculate greatest common denominator of frame lengths from all transforms in the pipeline
    long frameSizeGcd = Util.gcd(adjacencyMap.keySet().stream().map(Transform::watermarkFrameSize).filter(frameSize -> frameSize > 0).mapToLong(i -> i).toArray());
    WatermarkEmissionPolicy emitPolicy = frameSizeGcd > 0 ? emitByFrame(tumblingWinPolicy(frameSizeGcd)) : noThrottling();
    // Replace emission policy
    for (Transform transform : adjacencyMap.keySet()) {
        if (transform instanceof StreamSourceTransform) {
            StreamSourceTransform t = (StreamSourceTransform) transform;
            if (t.getWmParams() != null) {
                t.setWmGenerationParams(t.getWmParams().withEmitPolicy(emitPolicy));
            }
        } else if (transform instanceof TimestampTransform) {
            TimestampTransform t = (TimestampTransform) transform;
            t.setWmGenerationParams(t.getWmGenParams().withEmitPolicy(emitPolicy));
        }
    }
    Iterable<Transform> sorted = topologicalSort(adjacencyMap, Object::toString);
    for (Transform transform : sorted) {
        transform.addToDag(this);
    }
    return dag;
}

Example 2 with StreamSourceTransform

use of com.hazelcast.jet.impl.pipeline.transform.StreamSourceTransform in project hazelcast by hazelcast.

the class StreamSqlConnector method fullScanReader.

@Nonnull
@Override
public Vertex fullScanReader(@Nonnull DAG dag, @Nonnull Table table0, @Nullable Expression<Boolean> predicate, @Nonnull List<Expression<?>> projections, @Nullable FunctionEx<ExpressionEvalContext, EventTimePolicy<JetSqlRow>> eventTimePolicyProvider) {
    if (eventTimePolicyProvider != null) {
        throw QueryException.error("Ordering functions are not supported on top of " + TYPE_NAME + " mappings");
    }
    StreamTable table = (StreamTable) table0;
    StreamSourceTransform<JetSqlRow> source = (StreamSourceTransform<JetSqlRow>) table.items(predicate, projections);
    ProcessorMetaSupplier pms = source.metaSupplierFn.apply(EventTimePolicy.noEventTime());
    return dag.newUniqueVertex(table.toString(), pms);
}

Example 3 with StreamSourceTransform

use of com.hazelcast.jet.impl.pipeline.transform.StreamSourceTransform in project hazelcast by hazelcast.

the class Planner method createDag.

@SuppressWarnings("rawtypes")
DAG createDag(Context context) {
    pipeline.makeNamesUnique();
    Map<Transform, List<Transform>> adjacencyMap = pipeline.adjacencyMap();
    validateNoLeakage(adjacencyMap);
    checkTopologicalSort(adjacencyMap.entrySet());
    // Find the greatest common denominator of all frame lengths
    // appearing in the pipeline
    long frameSizeGcd = Util.gcd(adjacencyMap.keySet().stream().map(Transform::preferredWatermarkStride).filter(frameSize -> frameSize > 0).mapToLong(i -> i).toArray());
    if (frameSizeGcd == 0) {
        // even if there are no window aggregations, we want the watermarks for latency debugging
        frameSizeGcd = MAXIMUM_WATERMARK_GAP;
    }
    if (frameSizeGcd > MAXIMUM_WATERMARK_GAP) {
        frameSizeGcd = Util.gcd(frameSizeGcd, MAXIMUM_WATERMARK_GAP);
    }
    LoggingUtil.logFine(LOGGER, "Watermarks in the pipeline will be throttled to %d", frameSizeGcd);
    // Update watermark throttling frame length on all transforms with the determined length
    for (Transform transform : adjacencyMap.keySet()) {
        if (transform instanceof StreamSourceTransform) {
            StreamSourceTransform t = (StreamSourceTransform) transform;
            EventTimePolicy policy = t.getEventTimePolicy();
            if (policy != null) {
                t.setEventTimePolicy(withFrameSize(policy, frameSizeGcd));
            }
        } else if (transform instanceof TimestampTransform) {
            TimestampTransform t = (TimestampTransform) transform;
            t.setEventTimePolicy(withFrameSize(t.getEventTimePolicy(), frameSizeGcd));
        }
    }
    // fuse subsequent map/filter/flatMap transforms into one
    Map<Transform, List<Transform>> originalParents = new HashMap<>();
    List<Transform> transforms = new ArrayList<>(adjacencyMap.keySet());
    for (int i = 0; i < transforms.size(); i++) {
        Transform transform = transforms.get(i);
        List<Transform> chain = findFusableChain(transform, adjacencyMap);
        if (chain == null) {
            continue;
        }
        // remove transforms in the chain and replace the parent with a fused transform
        transforms.removeAll(chain.subList(1, chain.size()));
        Transform fused = fuseFlatMapTransforms(chain);
        transforms.set(i, fused);
        Transform lastInChain = chain.get(chain.size() - 1);
        for (Transform downstream : adjacencyMap.get(lastInChain)) {
            originalParents.put(downstream, new ArrayList<>(downstream.upstream()));
            downstream.upstream().replaceAll(p -> p == lastInChain ? fused : p);
        }
    }
    for (Transform transform : transforms) {
        transform.addToDag(this, context);
    }
    // restore original parents
    for (Entry<Transform, List<Transform>> en : originalParents.entrySet()) {
        List<Transform> upstream = en.getKey().upstream();
        for (int i = 0; i < upstream.size(); i++) {
            en.getKey().upstream().set(i, en.getValue().get(i));
        }
    }
    return dag;
}