use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.
the class WatermarkCoalescer_IntegrationTest method createDag.
private static DAG createDag(Mode mode, List<Object> input1, List<Object> input2) {
DAG dag = new DAG();
Vertex mapWmToString = dag.newVertex("mapWmToString", MapWatermarksToString::new).localParallelism(1);
Vertex sink = dag.newVertex("sink", writeListP("sinkList")).localParallelism(1);
dag.edge(between(mapWmToString, sink));
switch(mode) {
case TWO_EDGES:
Vertex edge1 = dag.newVertex("edge1", ListSource.supplier(input1)).localParallelism(1);
Vertex edge2 = dag.newVertex("edge2", ListSource.supplier(input2)).localParallelism(1);
dag.edge(from(edge1).to(mapWmToString, 0));
dag.edge(from(edge2).to(mapWmToString, 1));
break;
case TWO_QUEUES:
Vertex edge = dag.newVertex("edge", ListSource.supplier(input1, input2)).localParallelism(2);
dag.edge(between(edge, mapWmToString));
break;
default:
throw new IllegalArgumentException(mode.toString());
}
return dag;
}
use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.
the class HashJoinTransform method addToDag.
// --------- ---------- ----------
// | primary | | joined-1 | | joined-2 |
// --------- ---------- ----------
// | | |
// | distributed distributed
// | broadcast broadcast
// | v v
// | ------------- -------------
// | | collector-1 | | collector-2 |
// | ------------- -------------
// | | |
// | local local
// local broadcast broadcast
// unicast prioritized prioritized
// ordinal 0 ordinal 1 ordinal 2
// \ | |
// ----------------\ | /----------------/
// v v v
// --------
// | joiner |
// --------
@Override
@SuppressWarnings("unchecked")
public void addToDag(Planner p) {
String namePrefix = p.uniqueVertexName(this.name(), "");
PlannerVertex primary = p.xform2vertex.get(this.upstream().get(0));
List keyFns = this.clauses.stream().map(JoinClause::leftKeyFn).collect(toList());
List<Tag> tags = this.tags;
DistributedBiFunction mapToOutputBiFn = this.mapToOutputBiFn;
DistributedTriFunction mapToOutputTriFn = this.mapToOutputTriFn;
Vertex joiner = p.addVertex(this, namePrefix + "-joiner", localParallelism(), () -> new HashJoinP<>(keyFns, tags, mapToOutputBiFn, mapToOutputTriFn)).v;
p.dag.edge(from(primary.v, primary.nextAvailableOrdinal()).to(joiner, 0));
String collectorName = namePrefix + "-collector";
int collectorOrdinal = 1;
for (Transform fromTransform : tailList(this.upstream())) {
PlannerVertex fromPv = p.xform2vertex.get(fromTransform);
JoinClause<?, ?, ?, ?> clause = this.clauses.get(collectorOrdinal - 1);
DistributedFunction<Object, Object> getKeyFn = (DistributedFunction<Object, Object>) clause.rightKeyFn();
DistributedFunction<Object, Object> projectFn = (DistributedFunction<Object, Object>) clause.rightProjectFn();
Vertex collector = p.dag.newVertex(collectorName + collectorOrdinal, () -> new HashJoinCollectP(getKeyFn, projectFn));
collector.localParallelism(1);
p.dag.edge(from(fromPv.v, fromPv.nextAvailableOrdinal()).to(collector, 0).distributed().broadcast());
p.dag.edge(from(collector, 0).to(joiner, collectorOrdinal).broadcast().priority(-1));
collectorOrdinal++;
}
}
use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.
the class StreamSourceTransform method addToDag.
@Override
public void addToDag(Planner p) {
WatermarkGenerationParams<T> params = emitsJetEvents() ? wmParams : noWatermarks();
if (supportsWatermarks || !emitsJetEvents()) {
p.addVertex(this, p.uniqueVertexName(name(), ""), localParallelism(), metaSupplierFn.apply(params));
} else {
// ------------
// | sourceP |
// ------------
// |
// isolated
// v
// -------------
// | insertWMP |
// -------------
String v1name = p.uniqueVertexName(name(), "");
Vertex v1 = p.dag.newVertex(v1name, metaSupplierFn.apply(params)).localParallelism(localParallelism());
PlannerVertex pv2 = p.addVertex(this, v1name + "-insertWM", localParallelism(), insertWatermarksP(params));
p.dag.edge(between(v1, pv2.v).isolated());
}
}
use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.
the class Reducers method buildAccumulator.
private static <T> Vertex buildAccumulator(DAG dag, Pipe<? extends T> upstream, BinaryOperator<T> accumulator, T identity) {
Vertex reduce = reduceVertex(accumulator, identity);
dag.vertex(reduce);
Vertex previous = upstream.buildDAG(dag);
if (previous != reduce) {
dag.edge(between(previous, reduce));
}
return reduce;
}
use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.
the class Reducers method buildMappingAccumulator.
private static <T, U> Vertex buildMappingAccumulator(DAG dag, Pipe<? extends T> upstream, U identity, BiFunction<U, ? super T, U> accumulator) {
Vertex reduce = dag.newVertex("reduce", () -> new AccumulateP<>(accumulator, identity));
Vertex previous = upstream.buildDAG(dag);
if (previous != reduce) {
dag.edge(between(previous, reduce));
}
return reduce;
}
Aggregations