Examples with PartitionedProcessorTransform com.hazelcast.jet.impl.pipeline.transform.PartitionedProcessorTransform used on opensource projects

Example 1 with PartitionedProcessorTransform

use of com.hazelcast.jet.impl.pipeline.transform.PartitionedProcessorTransform in project hazelcast by hazelcast.

the class ComputeStageImplBase method attachMapUsingPartitionedServiceAsyncBatched.

@Nonnull
@SuppressWarnings({ "unchecked", "rawtypes" })
<S, K, R, RET> RET attachMapUsingPartitionedServiceAsyncBatched(@Nonnull ServiceFactory<?, S> serviceFactory, int maxBatchSize, @Nonnull FunctionEx<? super T, ? extends K> partitionKeyFn, @Nonnull BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<List<R>>> mapAsyncFn) {
    checkSerializable(mapAsyncFn, "mapAsyncFn");
    checkSerializable(partitionKeyFn, "partitionKeyFn");
    serviceFactory = moveAttachedFilesToPipeline(serviceFactory);
    BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<List<Traverser<R>>>> flatMapAsyncFn = (s, items) -> mapAsyncFn.apply(s, items).thenApply(list -> toList(list, Traversers::singleton));
    BiFunctionEx adaptedFlatMapFn = fnAdapter.adaptFlatMapUsingServiceAsyncBatchedFn(flatMapAsyncFn);
    FunctionEx adaptedPartitionKeyFn = fnAdapter.adaptKeyFn(partitionKeyFn);
    // Here we flatten the result from List<Traverser<R>> to Traverser<R>.
    // The former is used in pipeline API, the latter in core API.
    BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<Traverser<R>>> flattenedFn = (svc, items) -> {
        // R might actually be JetEvent<R> -- we can't represent this with static types
        CompletableFuture<List<Traverser<R>>> f = (CompletableFuture<List<Traverser<R>>>) adaptedFlatMapFn.apply(svc, items);
        return f.thenApply(res -> traverseIterable(res).flatMap(Function.identity()));
    };
    PartitionedProcessorTransform processorTransform = flatMapUsingServiceAsyncBatchedPartitionedTransform(transform, "map", serviceFactory, MAX_CONCURRENT_ASYNC_BATCHES, maxBatchSize, flattenedFn, adaptedPartitionKeyFn);
    return attach(processorTransform, fnAdapter);
}

Example 2 with PartitionedProcessorTransform

use of com.hazelcast.jet.impl.pipeline.transform.PartitionedProcessorTransform in project hazelcast by hazelcast.

the class ComputeStageImplBase method attachMapUsingPartitionedServiceAsync.

@Nonnull
@SuppressWarnings({ "unchecked", "rawtypes" })
<S, K, R, RET> RET attachMapUsingPartitionedServiceAsync(@Nonnull ServiceFactory<?, S> serviceFactory, int maxConcurrentOps, boolean preserveOrder, @Nonnull FunctionEx<? super T, ? extends K> partitionKeyFn, @Nonnull BiFunctionEx<? super S, ? super T, ? extends CompletableFuture<R>> mapAsyncFn) {
    checkSerializable(mapAsyncFn, "mapAsyncFn");
    checkSerializable(partitionKeyFn, "partitionKeyFn");
    serviceFactory = moveAttachedFilesToPipeline(serviceFactory);
    BiFunctionEx<? super S, ? super T, ? extends CompletableFuture<Traverser<R>>> flatMapAsyncFn = (s, t) -> mapAsyncFn.apply(s, t).thenApply(Traversers::singleton);
    BiFunctionEx adaptedFlatMapFn = fnAdapter.adaptFlatMapUsingServiceAsyncFn(flatMapAsyncFn);
    FunctionEx adaptedPartitionKeyFn = fnAdapter.adaptKeyFn(partitionKeyFn);
    PartitionedProcessorTransform processorTransform = flatMapUsingServiceAsyncPartitionedTransform(transform, "map", serviceFactory, maxConcurrentOps, preserveOrder, adaptedFlatMapFn, adaptedPartitionKeyFn);
    return attach(processorTransform, fnAdapter);
}