Examples with Traverser com.hazelcast.jet.Traverser used on opensource projects

Example 1 with Traverser

use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.

the class ComputeStageImplBase method attachMapUsingPartitionedServiceAsyncBatched.

@Nonnull
@SuppressWarnings({ "unchecked", "rawtypes" })
<S, K, R, RET> RET attachMapUsingPartitionedServiceAsyncBatched(@Nonnull ServiceFactory<?, S> serviceFactory, int maxBatchSize, @Nonnull FunctionEx<? super T, ? extends K> partitionKeyFn, @Nonnull BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<List<R>>> mapAsyncFn) {
    checkSerializable(mapAsyncFn, "mapAsyncFn");
    checkSerializable(partitionKeyFn, "partitionKeyFn");
    serviceFactory = moveAttachedFilesToPipeline(serviceFactory);
    BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<List<Traverser<R>>>> flatMapAsyncFn = (s, items) -> mapAsyncFn.apply(s, items).thenApply(list -> toList(list, Traversers::singleton));
    BiFunctionEx adaptedFlatMapFn = fnAdapter.adaptFlatMapUsingServiceAsyncBatchedFn(flatMapAsyncFn);
    FunctionEx adaptedPartitionKeyFn = fnAdapter.adaptKeyFn(partitionKeyFn);
    // Here we flatten the result from List<Traverser<R>> to Traverser<R>.
    // The former is used in pipeline API, the latter in core API.
    BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<Traverser<R>>> flattenedFn = (svc, items) -> {
        // R might actually be JetEvent<R> -- we can't represent this with static types
        CompletableFuture<List<Traverser<R>>> f = (CompletableFuture<List<Traverser<R>>>) adaptedFlatMapFn.apply(svc, items);
        return f.thenApply(res -> traverseIterable(res).flatMap(Function.identity()));
    };
    PartitionedProcessorTransform processorTransform = flatMapUsingServiceAsyncBatchedPartitionedTransform(transform, "map", serviceFactory, MAX_CONCURRENT_ASYNC_BATCHES, maxBatchSize, flattenedFn, adaptedPartitionKeyFn);
    return attach(processorTransform, fnAdapter);
}

Example 2 with Traverser

use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.

the class ComputeStageImplBase method attachMapUsingServiceAsyncBatched.

@Nonnull
@SuppressWarnings({ "unchecked", "rawtypes" })
<S, R, RET> RET attachMapUsingServiceAsyncBatched(@Nonnull ServiceFactory<?, S> serviceFactory, int maxBatchSize, @Nonnull BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<List<Traverser<R>>>> flatMapAsyncBatchedFn) {
    checkSerializable(flatMapAsyncBatchedFn, "mapAsyncBatchedFn");
    serviceFactory = moveAttachedFilesToPipeline(serviceFactory);
    BiFunctionEx adaptedFn = fnAdapter.adaptFlatMapUsingServiceAsyncBatchedFn(flatMapAsyncBatchedFn);
    // Here we flatten the result from List<Traverser<R>> to Traverser<R>.
    // The former is used in pipeline API, the latter in core API.
    BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<Traverser<R>>> flattenedFn = (svc, items) -> {
        // R might actually be JetEvent<R> -- we can't represent this with static types
        CompletableFuture<List<Traverser<R>>> f = (CompletableFuture<List<Traverser<R>>>) adaptedFn.apply(svc, items);
        return f.thenApply(res -> traverseIterable(res).flatMap(Function.identity()));
    };
    ProcessorTransform processorTransform = flatMapUsingServiceAsyncBatchedTransform(transform, "map", serviceFactory, MAX_CONCURRENT_ASYNC_BATCHES, maxBatchSize, flattenedFn);
    return attach(processorTransform, fnAdapter);
}

Example 3 with Traverser

use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.

the class CdcSourceP method complete.

@Override
public boolean complete() {
    if (!emitFromTraverser(traverser)) {
        return false;
    }
    if (reconnectTracker.needsToWait()) {
        return false;
    }
    if (!isConnectionUp()) {
        return false;
    }
    if (snapshotInProgress) {
        return false;
    }
    try {
        if (!snapshotting && commitPeriod > 0) {
            long currentTime = System.nanoTime();
            if (currentTime - lastCommitTime > commitPeriod) {
                task.commit();
                lastCommitTime = currentTime;
            }
        }
        List<SourceRecord> records = task.poll();
        if (records == null || records.isEmpty()) {
            traverser = eventTimeMapper.flatMapIdle();
            emitFromTraverser(traverser);
            return false;
        }
        for (SourceRecord record : records) {
            Map<String, ?> partition = record.sourcePartition();
            Map<String, ?> offset = record.sourceOffset();
            state.setOffset(partition, offset);
            task.commitRecord(record);
        }
        if (!snapshotting && commitPeriod == 0) {
            task.commit();
        }
        traverser = Traversers.traverseIterable(records).flatMap(record -> {
            T t = map(record);
            return t == null ? Traversers.empty() : eventTimeMapper.flatMapEvent(t, 0, extractTimestamp(record));
        });
        emitFromTraverser(traverser);
    } catch (InterruptedException ie) {
        logger.warning("Interrupted while waiting for data");
        Thread.currentThread().interrupt();
    } catch (RuntimeException re) {
        reconnect(re);
    }
    return false;
}

Example 4 with Traverser

use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.

the class AsyncTransformUsingServiceBatchP_IntegrationTest method stressTestInt.

private void stressTestInt(boolean restart) {
    /*
        This is a stress test of the cooperative emission using the DAG api. Only through DAG
        API we can configure edge queue sizes, which we use to cause more trouble for the
        cooperative emission.
         */
    // add more input to the source map
    int numItems = 10_000;
    journaledMap.putAll(IntStream.range(NUM_ITEMS, numItems).boxed().collect(toMap(i -> i, i -> i)));
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", throttle(streamMapP(journaledMap.getName(), alwaysTrue(), EventJournalMapEvent::getNewValue, START_FROM_OLDEST, eventTimePolicy(i -> (long) ((Integer) i), WatermarkPolicy.limitingLag(10), 10, 0, 0)), 5000));
    BiFunctionEx<ExecutorService, List<Integer>, CompletableFuture<Traverser<String>>> flatMapAsyncFn = transformNotPartitionedFn(i -> traverseItems(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5")).andThen(r -> r.thenApply(results -> traverseIterable(results).flatMap(Function.identity())));
    ProcessorSupplier processorSupplier = AsyncTransformUsingServiceBatchedP.supplier(serviceFactory, DEFAULT_MAX_CONCURRENT_OPS, 128, flatMapAsyncFn);
    Vertex map = dag.newVertex("map", processorSupplier).localParallelism(2);
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP(sinkList.getName()));
    // Use a shorter queue to not block the barrier from the source for too long due to
    // the backpressure from the slow mapper
    EdgeConfig edgeToMapperConfig = new EdgeConfig().setQueueSize(128);
    // Use a shorter queue on output from the mapper so that we experience backpressure
    // from the sink
    EdgeConfig edgeFromMapperConfig = new EdgeConfig().setQueueSize(10);
    dag.edge(between(source, map).setConfig(edgeToMapperConfig)).edge(between(map, sink).setConfig(edgeFromMapperConfig));
    Job job = instance().getJet().newJob(dag, jobConfig);
    for (int i = 0; restart && i < 5; i++) {
        assertNotNull(job);
        assertTrueEventually(() -> {
            JobStatus status = job.getStatus();
            assertTrue("status=" + status, status == RUNNING || status == COMPLETED);
        });
        sleepMillis(100);
        try {
            job.restart();
        } catch (IllegalStateException e) {
            assertTrue(e.toString(), e.getMessage().startsWith("Cannot RESTART_GRACEFUL"));
            break;
        }
    }
    assertResult(i -> Stream.of(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5"), numItems);
}

Example 5 with Traverser

use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.

the class AsyncTransformUsingServicePTest method test_futuresCompletedInSeparateThread.

@Test
public void test_futuresCompletedInSeparateThread() {
    TestSupport.verifyProcessor(getSupplier((ctx, item) -> {
        CompletableFuture<Traverser<String>> f = new CompletableFuture<>();
        spawn(() -> f.complete(traverseItems(item + "-1", item + "-2")));
        return f;
    })).hazelcastInstance(instance()).input(asList("a", "b", new Watermark(10))).outputChecker((expected, actual) -> actual.equals(asList("a-1", "a-2", "b-1", "b-2", wm(10))) || !ordered && actual.equals(asList("b-1", "b-2", "a-1", "a-2", wm(10)))).disableProgressAssertion().expectOutput(singletonList("<see code>"));
}