Search in sources :

Example 11 with Traverser

use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.

the class Planner method fuseFlatMapTransforms.

@SuppressWarnings("rawtypes")
private static Transform fuseFlatMapTransforms(List<Transform> chain) {
    assert chain.size() > 1 : "chain.size()=" + chain.size();
    assert chain.get(0).upstream().size() == 1;
    int lastFlatMap = 0;
    FunctionEx<Object, Traverser> flatMapFn = null;
    for (int i = 0; i < chain.size(); i++) {
        if (chain.get(i) instanceof FlatMapTransform) {
            FunctionEx<Object, Traverser> function = ((FlatMapTransform) chain.get(i)).flatMapFn();
            FunctionEx<Object, Object> inputMapFn = mergeMapFunctions(chain.subList(lastFlatMap, i));
            if (inputMapFn != null) {
                flatMapFn = flatMapFn == null ? (Object t) -> {
                    Object mappedValue = inputMapFn.apply(t);
                    return mappedValue != null ? function.apply(mappedValue) : Traversers.empty();
                } : flatMapFn.andThen(r -> r.map(inputMapFn).flatMap(function));
            } else {
                flatMapFn = flatMapFn == null ? function::apply : flatMapFn.andThen(r -> r.flatMap(function));
            }
            lastFlatMap = i + 1;
        }
    }
    FunctionEx trailingMapFn = mergeMapFunctions(chain.subList(lastFlatMap, chain.size()));
    String name = chain.stream().map(Transform::name).collect(Collectors.joining(", ", "fused(", ")"));
    Transform fused;
    if (flatMapFn == null) {
        fused = new MapTransform(name, chain.get(0).upstream().get(0), trailingMapFn);
    } else {
        if (trailingMapFn != null) {
            flatMapFn = flatMapFn.andThen(t -> t.map(trailingMapFn));
        }
        fused = new FlatMapTransform(name, chain.get(0).upstream().get(0), flatMapFn);
    }
    // if the first stage of the chain is rebalanced, then we set
    // the rebalance flag of the created fused stage. Only consider
    // the case when first element of the chain is rebalanced
    // because there isn't any other case. If any stage in the
    // middle includes rebalance, then those stages are not fused
    // by findFusableChain().
    fused.setRebalanceInput(0, chain.get(0).shouldRebalanceInput(0));
    return fused;
}
Also used : Traverser(com.hazelcast.jet.Traverser) LoggingUtil(com.hazelcast.jet.impl.util.LoggingUtil) Processor(com.hazelcast.jet.core.Processor) HashMap(java.util.HashMap) ObjIntConsumer(java.util.function.ObjIntConsumer) EventTimePolicy(com.hazelcast.jet.core.EventTimePolicy) ArrayList(java.util.ArrayList) SinkTransform(com.hazelcast.jet.impl.pipeline.transform.SinkTransform) TopologicalSorter.checkTopologicalSort(com.hazelcast.jet.impl.TopologicalSorter.checkTopologicalSort) ILogger(com.hazelcast.logging.ILogger) Map(java.util.Map) MapTransform(com.hazelcast.jet.impl.pipeline.transform.MapTransform) TimestampTransform(com.hazelcast.jet.impl.pipeline.transform.TimestampTransform) Edge.from(com.hazelcast.jet.core.Edge.from) DAG(com.hazelcast.jet.core.DAG) Edge(com.hazelcast.jet.core.Edge) Nonnull(javax.annotation.Nonnull) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) FunctionEx(com.hazelcast.function.FunctionEx) Logger(com.hazelcast.logging.Logger) Util.toList(com.hazelcast.jet.impl.util.Util.toList) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) Traversers(com.hazelcast.jet.Traversers) Collectors(java.util.stream.Collectors) SupplierEx(com.hazelcast.function.SupplierEx) Transform(com.hazelcast.jet.impl.pipeline.transform.Transform) Consumer(java.util.function.Consumer) Vertex(com.hazelcast.jet.core.Vertex) Util(com.hazelcast.jet.impl.util.Util) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) Entry(java.util.Map.Entry) StreamSourceTransform(com.hazelcast.jet.impl.pipeline.transform.StreamSourceTransform) EventTimePolicy.eventTimePolicy(com.hazelcast.jet.core.EventTimePolicy.eventTimePolicy) FlatMapTransform(com.hazelcast.jet.impl.pipeline.transform.FlatMapTransform) Context(com.hazelcast.jet.impl.pipeline.PipelineImpl.Context) Traverser(com.hazelcast.jet.Traverser) FunctionEx(com.hazelcast.function.FunctionEx) MapTransform(com.hazelcast.jet.impl.pipeline.transform.MapTransform) FlatMapTransform(com.hazelcast.jet.impl.pipeline.transform.FlatMapTransform) SinkTransform(com.hazelcast.jet.impl.pipeline.transform.SinkTransform) MapTransform(com.hazelcast.jet.impl.pipeline.transform.MapTransform) TimestampTransform(com.hazelcast.jet.impl.pipeline.transform.TimestampTransform) Transform(com.hazelcast.jet.impl.pipeline.transform.Transform) StreamSourceTransform(com.hazelcast.jet.impl.pipeline.transform.StreamSourceTransform) FlatMapTransform(com.hazelcast.jet.impl.pipeline.transform.FlatMapTransform) FlatMapTransform(com.hazelcast.jet.impl.pipeline.transform.FlatMapTransform)

Example 12 with Traverser

use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.

the class ElasticSourceP method init.

@Override
protected void init(@Nonnull Context context) throws Exception {
    super.init(context);
    logger = context.logger();
    logger.fine("init");
    client = configuration.clientFn().get();
    SearchRequest sr = configuration.searchRequestFn().get();
    sr.scroll(configuration.scrollKeepAlive());
    if (configuration.isSlicingEnabled()) {
        if (configuration.isCoLocatedReadingEnabled()) {
            int sliceId = context.localProcessorIndex();
            int totalSlices = context.localParallelism();
            if (totalSlices > 1) {
                logger.fine("Slice id=" + sliceId + ", max=" + totalSlices);
                sr.source().slice(new SliceBuilder(sliceId, totalSlices));
            }
        } else {
            int sliceId = context.globalProcessorIndex();
            int totalSlices = context.totalParallelism();
            if (totalSlices > 1) {
                logger.fine("Slice id=" + sliceId + ", max=" + totalSlices);
                sr.source().slice(new SliceBuilder(sliceId, totalSlices));
            }
        }
    }
    if (configuration.isCoLocatedReadingEnabled()) {
        logger.fine("Assigned shards: " + shards);
        if (shards.isEmpty()) {
            traverser = Traversers.empty();
            return;
        }
        Node node = createLocalElasticNode();
        client.getLowLevelClient().setNodes(singleton(node));
        String preference = "_shards:" + shards.stream().map(shard -> String.valueOf(shard.getShard())).collect(joining(",")) + "|_only_local";
        sr.preference(preference);
    }
    scrollTraverser = new ElasticScrollTraverser(configuration, client, sr, logger);
    traverser = scrollTraverser.map(configuration.mapToItemFn());
}
Also used : AbstractProcessor(com.hazelcast.jet.core.AbstractProcessor) Traverser(com.hazelcast.jet.Traverser) SearchHits(org.elasticsearch.search.SearchHits) ClearScrollRequest(org.elasticsearch.action.search.ClearScrollRequest) SearchRequest(org.elasticsearch.action.search.SearchRequest) Node(org.elasticsearch.client.Node) ClearScrollResponse(org.elasticsearch.action.search.ClearScrollResponse) JetException(com.hazelcast.jet.JetException) Collections.singleton(java.util.Collections.singleton) ILogger(com.hazelcast.logging.ILogger) Objects.requireNonNull(java.util.Objects.requireNonNull) SearchResponse(org.elasticsearch.action.search.SearchResponse) RequestOptions(org.elasticsearch.client.RequestOptions) Nonnull(javax.annotation.Nonnull) RetryUtils.withRetry(com.hazelcast.jet.elastic.impl.RetryUtils.withRetry) SearchHit(org.elasticsearch.search.SearchHit) FunctionEx(com.hazelcast.function.FunctionEx) ActionRequest(org.elasticsearch.action.ActionRequest) SliceBuilder(org.elasticsearch.search.slice.SliceBuilder) Traversers(com.hazelcast.jet.Traversers) RestHighLevelClient(org.elasticsearch.client.RestHighLevelClient) Collectors.joining(java.util.stream.Collectors.joining) TotalHits(org.apache.lucene.search.TotalHits) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) SearchScrollRequest(org.elasticsearch.action.search.SearchScrollRequest) HttpHost(org.apache.http.HttpHost) SearchRequest(org.elasticsearch.action.search.SearchRequest) SliceBuilder(org.elasticsearch.search.slice.SliceBuilder) Node(org.elasticsearch.client.Node)

Example 13 with Traverser

use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.

the class CreateDagVisitor method onSlidingWindow.

public Vertex onSlidingWindow(SlidingWindowPhysicalRel rel) {
    int orderingFieldIndex = rel.orderingFieldIndex();
    FunctionEx<ExpressionEvalContext, SlidingWindowPolicy> windowPolicySupplier = rel.windowPolicyProvider();
    // this vertex is used only if there's no aggregation by a window bound
    Vertex vertex = dag.newUniqueVertex("Sliding-Window", flatMapUsingServiceP(ServiceFactories.nonSharedService(ctx -> {
        ExpressionEvalContext evalContext = ExpressionEvalContext.from(ctx);
        SlidingWindowPolicy windowPolicy = windowPolicySupplier.apply(evalContext);
        return row -> WindowUtils.addWindowBounds(row, orderingFieldIndex, windowPolicy);
    }), (BiFunctionEx<Function<JetSqlRow, Traverser<JetSqlRow>>, JetSqlRow, Traverser<JetSqlRow>>) Function::apply));
    connectInput(rel.getInput(), vertex, null);
    return vertex;
}
Also used : ExpressionEvalContext(com.hazelcast.sql.impl.expression.ExpressionEvalContext) Address(com.hazelcast.cluster.Address) Traverser(com.hazelcast.jet.Traverser) ExpressionValues(com.hazelcast.jet.sql.impl.opt.ExpressionValues) PlanObjectKey(com.hazelcast.sql.impl.optimizer.PlanObjectKey) Processors.mapP(com.hazelcast.jet.core.processor.Processors.mapP) SourceProcessors.convenientSourceP(com.hazelcast.jet.core.processor.SourceProcessors.convenientSourceP) QueryParameterMetadata(com.hazelcast.sql.impl.QueryParameterMetadata) Collections.singletonList(java.util.Collections.singletonList) BiFunctionEx(com.hazelcast.function.BiFunctionEx) HazelcastTable(com.hazelcast.jet.sql.impl.schema.HazelcastTable) ObjectArrayKey(com.hazelcast.jet.sql.impl.ObjectArrayKey) SlidingWindowPolicy(com.hazelcast.jet.core.SlidingWindowPolicy) SqlConnectorUtil(com.hazelcast.jet.sql.impl.connector.SqlConnectorUtil) AggregateOperation(com.hazelcast.jet.aggregate.AggregateOperation) Functions.entryKey(com.hazelcast.function.Functions.entryKey) SqlConnectorUtil.getJetSqlConnector(com.hazelcast.jet.sql.impl.connector.SqlConnectorUtil.getJetSqlConnector) DAG(com.hazelcast.jet.core.DAG) RootResultConsumerSink.rootResultConsumerSink(com.hazelcast.jet.sql.impl.processors.RootResultConsumerSink.rootResultConsumerSink) ServiceFactories(com.hazelcast.jet.pipeline.ServiceFactories) FunctionEx(com.hazelcast.function.FunctionEx) KeyedWindowResultFunction(com.hazelcast.jet.core.function.KeyedWindowResultFunction) Predicate(java.util.function.Predicate) Collections.emptyList(java.util.Collections.emptyList) Set(java.util.Set) ConsumerEx(com.hazelcast.function.ConsumerEx) IMapSqlConnector(com.hazelcast.jet.sql.impl.connector.map.IMapSqlConnector) ExpressionUtil(com.hazelcast.jet.sql.impl.ExpressionUtil) Processors.filterUsingServiceP(com.hazelcast.jet.core.processor.Processors.filterUsingServiceP) List(java.util.List) ExpressionEvalContext(com.hazelcast.sql.impl.expression.ExpressionEvalContext) Processors.mapUsingServiceP(com.hazelcast.jet.core.processor.Processors.mapUsingServiceP) Table(com.hazelcast.sql.impl.schema.Table) ComparatorEx(com.hazelcast.function.ComparatorEx) QueryDataType(com.hazelcast.sql.impl.type.QueryDataType) Processors(com.hazelcast.jet.core.processor.Processors) Processors.flatMapUsingServiceP(com.hazelcast.jet.core.processor.Processors.flatMapUsingServiceP) TimestampKind(com.hazelcast.jet.core.TimestampKind) Function(java.util.function.Function) JetSqlRow(com.hazelcast.sql.impl.row.JetSqlRow) HashSet(java.util.HashSet) Edge.from(com.hazelcast.jet.core.Edge.from) Edge(com.hazelcast.jet.core.Edge) Expression(com.hazelcast.sql.impl.expression.Expression) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) VertexWithInputConfig(com.hazelcast.jet.sql.impl.connector.SqlConnector.VertexWithInputConfig) SingleRel(org.apache.calcite.rel.SingleRel) Nullable(javax.annotation.Nullable) JetJoinInfo(com.hazelcast.jet.sql.impl.JetJoinInfo) NodeEngine(com.hazelcast.spi.impl.NodeEngine) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) RelNode(org.apache.calcite.rel.RelNode) Consumer(java.util.function.Consumer) Vertex(com.hazelcast.jet.core.Vertex) ToLongFunctionEx(com.hazelcast.function.ToLongFunctionEx) SqlHashJoinP(com.hazelcast.jet.sql.impl.processors.SqlHashJoinP) ConstantExpression(com.hazelcast.sql.impl.expression.ConstantExpression) Processors.sortP(com.hazelcast.jet.core.processor.Processors.sortP) DefaultSerializationServiceBuilder(com.hazelcast.internal.serialization.impl.DefaultSerializationServiceBuilder) WindowUtils(com.hazelcast.jet.sql.impl.aggregate.WindowUtils) Edge.between(com.hazelcast.jet.core.Edge.between) Vertex(com.hazelcast.jet.core.Vertex) KeyedWindowResultFunction(com.hazelcast.jet.core.function.KeyedWindowResultFunction) Function(java.util.function.Function) SlidingWindowPolicy(com.hazelcast.jet.core.SlidingWindowPolicy) Traverser(com.hazelcast.jet.Traverser) JetSqlRow(com.hazelcast.sql.impl.row.JetSqlRow) BiFunctionEx(com.hazelcast.function.BiFunctionEx)

Example 14 with Traverser

use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.

the class StreamKafkaP method saveToSnapshot.

@Override
public boolean saveToSnapshot() {
    if (!emitFromTraverser(traverser)) {
        return false;
    }
    if (snapshotTraverser == null) {
        Stream<Entry<BroadcastKey<?>, ?>> snapshotStream = offsets.entrySet().stream().flatMap(entry -> IntStream.range(0, entry.getValue().length).filter(partition -> entry.getValue()[partition] >= 0).mapToObj(partition -> {
            TopicPartition key = new TopicPartition(entry.getKey(), partition);
            long offset = entry.getValue()[partition];
            long watermark = eventTimeMapper.getWatermark(currentAssignment.get(key));
            return entry(broadcastKey(key), new long[] { offset, watermark });
        }));
        snapshotTraverser = traverseStream(snapshotStream).onFirstNull(() -> {
            snapshotTraverser = null;
            if (getLogger().isFinestEnabled()) {
                getLogger().finest("Finished saving snapshot." + " Saved offsets: " + offsets() + ", Saved watermarks: " + watermarks());
            }
        });
        if (processorIndex == 0) {
            Entry<BroadcastKey<?>, ?> partitionCountsItem = entry(broadcastKey(PARTITION_COUNTS_SNAPSHOT_KEY), topics.stream().collect(Collectors.toMap(topic -> topic, topic -> offsets.get(topic).length)));
            snapshotTraverser = snapshotTraverser.append(partitionCountsItem);
        }
    }
    return emitFromTraverserToSnapshot(snapshotTraverser);
}
Also used : IntStream(java.util.stream.IntStream) AbstractProcessor(com.hazelcast.jet.core.AbstractProcessor) Traverser(com.hazelcast.jet.Traverser) Arrays(java.util.Arrays) Processor(com.hazelcast.jet.core.Processor) Traversers.traverseStream(com.hazelcast.jet.Traversers.traverseStream) LoggingUtil.logFinest(com.hazelcast.jet.impl.util.LoggingUtil.logFinest) HashMap(java.util.HashMap) EventTimePolicy(com.hazelcast.jet.core.EventTimePolicy) KafkaProcessors(com.hazelcast.jet.kafka.KafkaProcessors) ConsumerRecords(org.apache.kafka.clients.consumer.ConsumerRecords) ArrayList(java.util.ArrayList) Traversers.traverseIterable(com.hazelcast.jet.Traversers.traverseIterable) Util.entry(com.hazelcast.jet.Util.entry) Duration(java.time.Duration) Map(java.util.Map) Nonnull(javax.annotation.Nonnull) EventTimeMapper(com.hazelcast.jet.core.EventTimeMapper) FunctionEx(com.hazelcast.function.FunctionEx) TopicPartition(org.apache.kafka.common.TopicPartition) TimeoutException(org.apache.kafka.common.errors.TimeoutException) Properties(java.util.Properties) Collection(java.util.Collection) Traversers(com.hazelcast.jet.Traversers) Collectors(java.util.stream.Collectors) SupplierEx(com.hazelcast.function.SupplierEx) BroadcastKey(com.hazelcast.jet.core.BroadcastKey) List(java.util.List) BroadcastKey.broadcastKey(com.hazelcast.jet.core.BroadcastKey.broadcastKey) Stream(java.util.stream.Stream) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) Entry(java.util.Map.Entry) SECONDS(java.util.concurrent.TimeUnit.SECONDS) KafkaConsumer(org.apache.kafka.clients.consumer.KafkaConsumer) Entry(java.util.Map.Entry) TopicPartition(org.apache.kafka.common.TopicPartition) BroadcastKey(com.hazelcast.jet.core.BroadcastKey)

Example 15 with Traverser

use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.

the class AppendableTraverserTest method test_flatMapperUsage.

@Test
public void test_flatMapperUsage() {
    // an instance of AppendableTraverser is repeatedly returned
    // from a flatMap function
    Traverser tt = Traversers.traverseItems(10, 20).flatMap(item -> {
        assertTrue(t.isEmpty());
        t.append(item);
        t.append(item + 1);
        return t;
    });
    assertEquals(10, tt.next());
    assertEquals(11, tt.next());
    assertEquals(20, tt.next());
    assertEquals(21, tt.next());
    assertNull(tt.next());
}
Also used : Traverser(com.hazelcast.jet.Traverser) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) QuickTest(com.hazelcast.test.annotation.QuickTest) Test(org.junit.Test)

Aggregations

Traverser (com.hazelcast.jet.Traverser)18 List (java.util.List)12 FunctionEx (com.hazelcast.function.FunctionEx)10 Nonnull (javax.annotation.Nonnull)10 Traversers (com.hazelcast.jet.Traversers)9 BiFunctionEx (com.hazelcast.function.BiFunctionEx)7 JetException (com.hazelcast.jet.JetException)7 Function (java.util.function.Function)7 Nullable (javax.annotation.Nullable)7 Traversers.traverseIterable (com.hazelcast.jet.Traversers.traverseIterable)6 EventTimePolicy.eventTimePolicy (com.hazelcast.jet.core.EventTimePolicy.eventTimePolicy)6 ProcessorMetaSupplier (com.hazelcast.jet.core.ProcessorMetaSupplier)6 CompletableFuture (java.util.concurrent.CompletableFuture)6 SupplierEx (com.hazelcast.function.SupplierEx)5 AbstractProcessor (com.hazelcast.jet.core.AbstractProcessor)5 ProcessorSupplier (com.hazelcast.jet.core.ProcessorSupplier)5 ComparatorEx (com.hazelcast.function.ComparatorEx)4 ToLongFunctionEx (com.hazelcast.function.ToLongFunctionEx)4 Traversers.traverseStream (com.hazelcast.jet.Traversers.traverseStream)4 DAG (com.hazelcast.jet.core.DAG)4