use of com.hazelcast.jet.impl.pipeline.transform.Transform in project hazelcast-jet by hazelcast.
the class GrAggBuilder method buildStream.
@SuppressWarnings("unchecked")
public <A, R, OUT> StreamStage<OUT> buildStream(@Nonnull AggregateOperation<A, ? extends R> aggrOp, @Nonnull KeyedWindowResultFunction<? super K, ? super R, OUT> mapToOutputFn) {
List<Transform> upstreamTransforms = upstreamStages.stream().map(s -> s.transform).collect(toList());
JetEventFunctionAdapter fnAdapter = ADAPT_TO_JET_EVENT;
// Avoided Stream API here due to static typing issues
List<DistributedFunction<?, ? extends K>> adaptedKeyFns = new ArrayList<>();
for (DistributedFunction keyFn : keyFns) {
adaptedKeyFns.add(adaptKeyFn(keyFn));
}
Transform transform = new WindowGroupTransform<K, A, R, JetEvent<OUT>>(upstreamTransforms, wDef, adaptedKeyFns, adaptAggregateOperation(aggrOp), fnAdapter.adaptKeyedWindowResultFn(mapToOutputFn));
pipelineImpl.connect(upstreamTransforms, transform);
return new StreamStageImpl<>(transform, fnAdapter, pipelineImpl);
}
use of com.hazelcast.jet.impl.pipeline.transform.Transform in project hazelcast-jet by hazelcast.
the class PipelineImpl method drainTo.
@Override
public <T> SinkStage drainTo(@Nonnull Sink<T> sink, GeneralStage<?>... stagesToDrain) {
if (stagesToDrain == null || stagesToDrain.length == 0) {
throw new IllegalArgumentException("No stages supplied to Pipeline.drainTo()");
}
List<Transform> upstream = Arrays.stream(stagesToDrain).map(s -> (AbstractStage) s).map(s -> s.transform).collect(toList());
int[] ordinalsToAdapt = IntStream.range(0, stagesToDrain.length).filter(i -> ((ComputeStageImplBase) stagesToDrain[i]).fnAdapter == ADAPT_TO_JET_EVENT).toArray();
SinkImpl sinkImpl = (SinkImpl) sink;
SinkTransform sinkTransform = new SinkTransform(sinkImpl, upstream, ordinalsToAdapt);
SinkStageImpl sinkStage = new SinkStageImpl(sinkTransform, this);
sinkImpl.onAssignToStage();
connect(upstream, sinkTransform);
return sinkStage;
}
use of com.hazelcast.jet.impl.pipeline.transform.Transform in project hazelcast-jet by hazelcast.
the class Planner method createDag.
DAG createDag() {
Map<Transform, List<Transform>> adjacencyMap = pipeline.adjacencyMap();
validateNoLeakage(adjacencyMap);
// Calculate greatest common denominator of frame lengths from all transforms in the pipeline
long frameSizeGcd = Util.gcd(adjacencyMap.keySet().stream().map(Transform::watermarkFrameSize).filter(frameSize -> frameSize > 0).mapToLong(i -> i).toArray());
WatermarkEmissionPolicy emitPolicy = frameSizeGcd > 0 ? emitByFrame(tumblingWinPolicy(frameSizeGcd)) : noThrottling();
// Replace emission policy
for (Transform transform : adjacencyMap.keySet()) {
if (transform instanceof StreamSourceTransform) {
StreamSourceTransform t = (StreamSourceTransform) transform;
if (t.getWmParams() != null) {
t.setWmGenerationParams(t.getWmParams().withEmitPolicy(emitPolicy));
}
} else if (transform instanceof TimestampTransform) {
TimestampTransform t = (TimestampTransform) transform;
t.setWmGenerationParams(t.getWmGenParams().withEmitPolicy(emitPolicy));
}
}
Iterable<Transform> sorted = topologicalSort(adjacencyMap, Object::toString);
for (Transform transform : sorted) {
transform.addToDag(this);
}
return dag;
}
use of com.hazelcast.jet.impl.pipeline.transform.Transform in project hazelcast by hazelcast.
the class ComputeStageImplBase method attachMapUsingPartitionedServiceAsyncBatched.
@Nonnull
@SuppressWarnings({ "unchecked", "rawtypes" })
<S, K, R, RET> RET attachMapUsingPartitionedServiceAsyncBatched(@Nonnull ServiceFactory<?, S> serviceFactory, int maxBatchSize, @Nonnull FunctionEx<? super T, ? extends K> partitionKeyFn, @Nonnull BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<List<R>>> mapAsyncFn) {
checkSerializable(mapAsyncFn, "mapAsyncFn");
checkSerializable(partitionKeyFn, "partitionKeyFn");
serviceFactory = moveAttachedFilesToPipeline(serviceFactory);
BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<List<Traverser<R>>>> flatMapAsyncFn = (s, items) -> mapAsyncFn.apply(s, items).thenApply(list -> toList(list, Traversers::singleton));
BiFunctionEx adaptedFlatMapFn = fnAdapter.adaptFlatMapUsingServiceAsyncBatchedFn(flatMapAsyncFn);
FunctionEx adaptedPartitionKeyFn = fnAdapter.adaptKeyFn(partitionKeyFn);
// Here we flatten the result from List<Traverser<R>> to Traverser<R>.
// The former is used in pipeline API, the latter in core API.
BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<Traverser<R>>> flattenedFn = (svc, items) -> {
// R might actually be JetEvent<R> -- we can't represent this with static types
CompletableFuture<List<Traverser<R>>> f = (CompletableFuture<List<Traverser<R>>>) adaptedFlatMapFn.apply(svc, items);
return f.thenApply(res -> traverseIterable(res).flatMap(Function.identity()));
};
PartitionedProcessorTransform processorTransform = flatMapUsingServiceAsyncBatchedPartitionedTransform(transform, "map", serviceFactory, MAX_CONCURRENT_ASYNC_BATCHES, maxBatchSize, flattenedFn, adaptedPartitionKeyFn);
return attach(processorTransform, fnAdapter);
}
use of com.hazelcast.jet.impl.pipeline.transform.Transform in project hazelcast by hazelcast.
the class ComputeStageImplBase method attachMapUsingServiceAsyncBatched.
@Nonnull
@SuppressWarnings({ "unchecked", "rawtypes" })
<S, R, RET> RET attachMapUsingServiceAsyncBatched(@Nonnull ServiceFactory<?, S> serviceFactory, int maxBatchSize, @Nonnull BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<List<Traverser<R>>>> flatMapAsyncBatchedFn) {
checkSerializable(flatMapAsyncBatchedFn, "mapAsyncBatchedFn");
serviceFactory = moveAttachedFilesToPipeline(serviceFactory);
BiFunctionEx adaptedFn = fnAdapter.adaptFlatMapUsingServiceAsyncBatchedFn(flatMapAsyncBatchedFn);
// Here we flatten the result from List<Traverser<R>> to Traverser<R>.
// The former is used in pipeline API, the latter in core API.
BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<Traverser<R>>> flattenedFn = (svc, items) -> {
// R might actually be JetEvent<R> -- we can't represent this with static types
CompletableFuture<List<Traverser<R>>> f = (CompletableFuture<List<Traverser<R>>>) adaptedFn.apply(svc, items);
return f.thenApply(res -> traverseIterable(res).flatMap(Function.identity()));
};
ProcessorTransform processorTransform = flatMapUsingServiceAsyncBatchedTransform(transform, "map", serviceFactory, MAX_CONCURRENT_ASYNC_BATCHES, maxBatchSize, flattenedFn);
return attach(processorTransform, fnAdapter);
}
Aggregations