Examples with Traverser com.hazelcast.jet.Traverser used on opensource projects

Example 6 with Traverser

use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.

the class AsyncTransformUsingServiceBatchedPTest method test_futuresCompletedInSeparateThread.

@Test
public void test_futuresCompletedInSeparateThread() {
    TestSupport.verifyProcessor(getSupplier((ctx, items) -> {
        CompletableFuture<Traverser<String>> f = new CompletableFuture<>();
        spawn(() -> f.complete(traverseIterable(items).flatMap(item -> traverseItems(item + "-1", item + "-2"))));
        return f;
    })).hazelcastInstance(instance()).input(asList("a", "b", new Watermark(10))).outputChecker((expected, actual) -> actual.equals(asList("a-1", "a-2", "b-1", "b-2", wm(10)))).disableProgressAssertion().expectOutput(singletonList("<see code>"));
}

Example 7 with Traverser

use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.

the class AsyncTransformUsingServiceP_IntegrationTest method stressTestInt.

private void stressTestInt(boolean restart) {
    /*
        This is a stress test of the cooperative emission using the DAG api. Only through DAG
        API we can configure edge queue sizes, which we use to cause more trouble for the
        cooperative emission.
         */
    // add more input to the source map
    int numItems = 10_000;
    journaledMap.putAll(IntStream.range(NUM_ITEMS, numItems).boxed().collect(toMap(i -> i, i -> i)));
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", throttle(streamMapP(journaledMap.getName(), alwaysTrue(), EventJournalMapEvent::getNewValue, START_FROM_OLDEST, eventTimePolicy(i -> (long) ((Integer) i), WatermarkPolicy.limitingLag(10), 10, 0, 0)), 5000));
    BiFunctionEx<ExecutorService, Integer, CompletableFuture<Traverser<String>>> flatMapAsyncFn = transformNotPartitionedFn(i -> traverseItems(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5"));
    ProcessorSupplier processorSupplier = ordered ? AsyncTransformUsingServiceOrderedP.supplier(serviceFactory, DEFAULT_MAX_CONCURRENT_OPS, flatMapAsyncFn) : AsyncTransformUsingServiceUnorderedP.supplier(serviceFactory, DEFAULT_MAX_CONCURRENT_OPS, flatMapAsyncFn, identity());
    Vertex map = dag.newVertex("map", processorSupplier).localParallelism(2);
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP(sinkList.getName()));
    // Use a shorter queue to not block the barrier from the source for too long due to
    // the backpressure from the slow mapper
    EdgeConfig edgeToMapperConfig = new EdgeConfig().setQueueSize(128);
    // Use a shorter queue on output from the mapper so that we experience backpressure
    // from the sink
    EdgeConfig edgeFromMapperConfig = new EdgeConfig().setQueueSize(10);
    dag.edge(between(source, map).setConfig(edgeToMapperConfig)).edge(between(map, sink).setConfig(edgeFromMapperConfig));
    Job job = instance().getJet().newJob(dag, jobConfig);
    for (int i = 0; restart && i < 5; i++) {
        assertJobStatusEventually(job, RUNNING);
        sleepMillis(100);
        job.restart();
    }
    assertResultEventually(i -> Stream.of(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5"), numItems);
}

Example 8 with Traverser

use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.

the class KinesisSourceP method runReaders.

private void runReaders() {
    if (!shardReaders.isEmpty()) {
        long currentTime = System.nanoTime();
        for (int i = 0; i < shardReaders.size(); i++) {
            int currentReader = nextReader;
            ShardReader reader = shardReaders.get(currentReader);
            nextReader = incrementCircular(currentReader, shardReaders.size());
            ShardReader.Result result = reader.probe(currentTime);
            if (ShardReader.Result.HAS_DATA.equals(result)) {
                Shard shard = reader.getShard();
                traverser = reader.clearData().flatMap(record -> {
                    T payload = projectionFn.apply(record, shard);
                    return eventTimeMapper.flatMapEvent(payload, currentReader, record.getApproximateArrivalTimestamp().getTime());
                });
                Long watermark = eventTimeMapper.getWatermark(currentReader);
                watermark = watermark < 0 ? null : watermark;
                shardStates.update(shard, reader.getLastSeenSeqNo(), watermark);
                emitFromTraverser(traverser);
                return;
            } else if (ShardReader.Result.CLOSED.equals(result)) {
                Shard shard = reader.getShard();
                logger.info("Shard " + shard.getShardId() + " of stream " + stream + " closed");
                shardStates.close(shard);
                nextReader = 0;
                traverser = removeShardReader(currentReader);
                emitFromTraverser(traverser);
                return;
            }
        }
    }
    traverser = eventTimeMapper.flatMapIdle();
    emitFromTraverser(traverser);
}

Example 9 with Traverser

use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.

the class ReadFilesP method init.

@Override
protected void init(@Nonnull Context context) {
    ILogger logger = context.logger();
    int processorIndex = sharedFileSystem ? context.globalProcessorIndex() : context.localProcessorIndex();
    int parallelism = sharedFileSystem ? context.totalParallelism() : context.localParallelism();
    traverser = new LocalFileTraverser<>(logger, directory, glob, ignoreFileNotFound, path -> shouldProcessEvent(path, parallelism, processorIndex), readFileFn);
}

Example 10 with Traverser

use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.

the class ComputeStageImplBase method attachMapUsingPartitionedServiceAsync.

@Nonnull
@SuppressWarnings({ "unchecked", "rawtypes" })
<S, K, R, RET> RET attachMapUsingPartitionedServiceAsync(@Nonnull ServiceFactory<?, S> serviceFactory, int maxConcurrentOps, boolean preserveOrder, @Nonnull FunctionEx<? super T, ? extends K> partitionKeyFn, @Nonnull BiFunctionEx<? super S, ? super T, ? extends CompletableFuture<R>> mapAsyncFn) {
    checkSerializable(mapAsyncFn, "mapAsyncFn");
    checkSerializable(partitionKeyFn, "partitionKeyFn");
    serviceFactory = moveAttachedFilesToPipeline(serviceFactory);
    BiFunctionEx<? super S, ? super T, ? extends CompletableFuture<Traverser<R>>> flatMapAsyncFn = (s, t) -> mapAsyncFn.apply(s, t).thenApply(Traversers::singleton);
    BiFunctionEx adaptedFlatMapFn = fnAdapter.adaptFlatMapUsingServiceAsyncFn(flatMapAsyncFn);
    FunctionEx adaptedPartitionKeyFn = fnAdapter.adaptKeyFn(partitionKeyFn);
    PartitionedProcessorTransform processorTransform = flatMapUsingServiceAsyncPartitionedTransform(transform, "map", serviceFactory, maxConcurrentOps, preserveOrder, adaptedFlatMapFn, adaptedPartitionKeyFn);
    return attach(processorTransform, fnAdapter);
}