use of com.hazelcast.jet.pipeline.GeneralStage in project hazelcast-jet by hazelcast.
the class PipelineImpl method drainTo.
@Override
public <T> SinkStage drainTo(@Nonnull Sink<T> sink, GeneralStage<?>... stagesToDrain) {
if (stagesToDrain == null || stagesToDrain.length == 0) {
throw new IllegalArgumentException("No stages supplied to Pipeline.drainTo()");
}
List<Transform> upstream = Arrays.stream(stagesToDrain).map(s -> (AbstractStage) s).map(s -> s.transform).collect(toList());
int[] ordinalsToAdapt = IntStream.range(0, stagesToDrain.length).filter(i -> ((ComputeStageImplBase) stagesToDrain[i]).fnAdapter == ADAPT_TO_JET_EVENT).toArray();
SinkImpl sinkImpl = (SinkImpl) sink;
SinkTransform sinkTransform = new SinkTransform(sinkImpl, upstream, ordinalsToAdapt);
SinkStageImpl sinkStage = new SinkStageImpl(sinkTransform, this);
sinkImpl.onAssignToStage();
connect(upstream, sinkTransform);
return sinkStage;
}
use of com.hazelcast.jet.pipeline.GeneralStage in project hazelcast by hazelcast.
the class PipelineImpl method writeTo.
@Nonnull
@Override
@SuppressWarnings({ "rawtypes", "unchecked" })
public <T> SinkStage writeTo(@Nonnull Sink<? super T> sink, @Nonnull GeneralStage<? extends T> stage0, @Nonnull GeneralStage<? extends T> stage1, @Nonnull GeneralStage<? extends T>... moreStages) {
List<GeneralStage> stages = new ArrayList<>(asList(moreStages));
stages.add(0, stage0);
stages.add(1, stage1);
List<Transform> upstream = stages.stream().map(s -> (AbstractStage) s).map(s -> s.transform).collect(toList());
int[] ordinalsToAdapt = IntStream.range(0, stages.size()).filter(i -> ((ComputeStageImplBase) stages.get(i)).fnAdapter == ADAPT_TO_JET_EVENT).toArray();
SinkImpl sinkImpl = (SinkImpl) sink;
SinkTransform sinkTransform = new SinkTransform(sinkImpl, upstream, ordinalsToAdapt);
SinkStageImpl sinkStage = new SinkStageImpl(sinkTransform, this);
sinkImpl.onAssignToStage();
connectGeneralStages(stages, sinkTransform);
return sinkStage;
}
use of com.hazelcast.jet.pipeline.GeneralStage in project hazelcast by hazelcast.
the class StageWithKeyAndWindowImpl method aggregate2.
@Nonnull
@Override
@SuppressWarnings("rawtypes")
public <T1, R> StreamStage<KeyedWindowResult<K, R>> aggregate2(@Nonnull StreamStageWithKey<T1, ? extends K> stage1, @Nonnull AggregateOperation2<? super T, ? super T1, ?, ? extends R> aggrOp) {
ensureJetEvents(computeStage, "This pipeline stage");
ComputeStageImplBase computeStage1 = ((StageWithGroupingBase) stage1).computeStage;
ensureJetEvents(computeStage1, "stage1");
Transform upstream1 = computeStage1.transform;
FunctionAdapter fnAdapter = ADAPT_TO_JET_EVENT;
return this.computeStage.attach(new WindowGroupTransform<K, R>(asList(this.computeStage.transform, upstream1), wDef, asList(fnAdapter.adaptKeyFn(keyFn()), fnAdapter.adaptKeyFn(stage1.keyFn())), adaptAggregateOperation2(aggrOp)), singletonList((GeneralStage<?>) computeStage1), fnAdapter);
}
use of com.hazelcast.jet.pipeline.GeneralStage in project hazelcast by hazelcast.
the class StageWithKeyAndWindowImpl method aggregate3.
@Nonnull
@Override
@SuppressWarnings("rawtypes")
public <T1, T2, R> StreamStage<KeyedWindowResult<K, R>> aggregate3(@Nonnull StreamStageWithKey<T1, ? extends K> stage1, @Nonnull StreamStageWithKey<T2, ? extends K> stage2, @Nonnull AggregateOperation3<? super T, ? super T1, ? super T2, ?, ? extends R> aggrOp) {
ComputeStageImplBase computeStage1 = ((StageWithGroupingBase) stage1).computeStage;
ComputeStageImplBase computeStage2 = ((StageWithGroupingBase) stage2).computeStage;
ensureJetEvents(computeStage, "This pipeline stage");
ensureJetEvents(computeStage1, "stage1");
ensureJetEvents(computeStage2, "stage2");
Transform transform1 = ((StageWithGroupingBase) stage1).computeStage.transform;
Transform transform2 = ((StageWithGroupingBase) stage2).computeStage.transform;
FunctionAdapter fnAdapter = ADAPT_TO_JET_EVENT;
return computeStage.attach(new WindowGroupTransform<K, R>(asList(computeStage.transform, transform1, transform2), wDef, asList(fnAdapter.adaptKeyFn(keyFn()), fnAdapter.adaptKeyFn(stage1.keyFn()), fnAdapter.adaptKeyFn(stage2.keyFn())), adaptAggregateOperation3(aggrOp)), asList((GeneralStage<?>) computeStage1, (GeneralStage<?>) computeStage2), fnAdapter);
}
use of com.hazelcast.jet.pipeline.GeneralStage in project hazelcast-jet by hazelcast.
the class AggBuilder method build.
@Nonnull
@SuppressWarnings("unchecked")
public <A, R, OUT, OUT_STAGE extends GeneralStage<OUT>> OUT_STAGE build(@Nonnull AggregateOperation<A, R> aggrOp, @Nonnull CreateOutStageFn<OUT, OUT_STAGE> createOutStageFn, @Nullable WindowResultFunction<? super R, ? extends OUT> mapToOutputFn) {
AggregateOperation adaptedAggrOp = wDef != null ? adaptAggregateOperation(aggrOp) : aggrOp;
List<Transform> upstreamTransforms = upstreamStages.stream().map(s -> ((AbstractStage) s).transform).collect(toList());
final Transform transform;
if (wDef != null) {
requireNonNull(mapToOutputFn, "wDef != null but mapToOutputFn == null");
transform = new WindowAggregateTransform<>(upstreamTransforms, wDef, adaptedAggrOp, mapToOutputFn);
} else {
transform = new AggregateTransform<>(upstreamTransforms, adaptedAggrOp);
}
OUT_STAGE attached = createOutStageFn.get(transform, DONT_ADAPT, pipelineImpl);
pipelineImpl.connect(upstreamTransforms, transform);
return attached;
}
Aggregations