Examples with Vertex com.hazelcast.jet.core.Vertex used on opensource projects

Example 6 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.

the class Reducers method execute.

private static <T> Optional<T> execute(StreamContext context, DAG dag, Vertex combiner) {
    String listName = uniqueListName();
    Vertex writeList = dag.newVertex("write-" + listName, SinkProcessors.writeListP(listName));
    dag.edge(between(combiner, writeList));
    IList<T> list = context.getJetInstance().getList(listName);
    executeJob(context, dag);
    if (list.isEmpty()) {
        list.destroy();
        return Optional.empty();
    }
    T result = list.get(0);
    list.destroy();
    return Optional.of(result);
}

Example 7 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.

the class BackpressureTest method testBackpressure.

@Test
public void testBackpressure() throws Exception {
    DAG dag = new DAG();
    final int member1Port = jet1.getCluster().getLocalMember().getAddress().getPort();
    final Member member2 = jet2.getCluster().getLocalMember();
    final int ptionOwnedByMember2 = jet1.getHazelcastInstance().getPartitionService().getPartitions().stream().filter(p -> p.getOwner().equals(member2)).map(Partition::getPartitionId).findAny().orElseThrow(() -> new RuntimeException("Can't find a partition owned by member " + jet2));
    Vertex source = dag.newVertex("source", ProcessorMetaSupplier.of((Address address) -> ProcessorSupplier.of(address.getPort() == member1Port ? GenerateP::new : noopP())));
    Vertex hiccup = dag.newVertex("hiccup", HiccupP::new);
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeMapP("counts"));
    dag.edge(between(source, hiccup).distributed().partitioned(wholeItem(), (x, y) -> ptionOwnedByMember2)).edge(between(hiccup, sink));
    jet1.newJob(dag).join();
    assertCounts(jet1.getMap("counts"));
}

Example 8 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.

the class AnyMatchReducer method reduce.

@Override
public Boolean reduce(StreamContext context, Pipe<? extends T> upstream) {
    String listName = uniqueListName();
    DAG dag = new DAG();
    Vertex previous = upstream.buildDAG(dag);
    Vertex anyMatch = dag.newVertex("any-match", () -> new AnyMatchP<>(predicate));
    Vertex writer = dag.newVertex("write-" + listName, SinkProcessors.writeListP(listName));
    dag.edge(between(previous, anyMatch)).edge(between(anyMatch, writer));
    executeJob(context, dag);
    IList<Boolean> results = context.getJetInstance().getList(listName);
    boolean result = anyMatch(results);
    results.destroy();
    return result;
}

Example 9 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.

the class BiConsumerCombinerReducer method reduce.

@Override
public R reduce(StreamContext context, Pipe<? extends T> upstream) {
    DAG dag = new DAG();
    Vertex accumulatorVertex = buildAccumulator(dag, upstream, supplier, accumulator);
    Vertex combinerVertex = buildCombiner(dag, accumulatorVertex, combiner);
    return execute(context, dag, combinerVertex, DistributedFunction.identity());
}

Example 10 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.

the class CollectorReducer method buildAccumulator.

static <T, R> Vertex buildAccumulator(DAG dag, Pipe<T> upstream, Supplier<R> supplier, BiConsumer<R, ? super T> accumulator) {
    Vertex accumulatorVertex = dag.newVertex("accumulator", () -> new CollectorAccumulateP<>(accumulator, supplier));
    if (upstream.isOrdered()) {
        accumulatorVertex.localParallelism(1);
    }
    Vertex previous = upstream.buildDAG(dag);
    if (previous != accumulatorVertex) {
        dag.edge(between(previous, accumulatorVertex));
    }
    return accumulatorVertex;
}