Examples with DAG com.hazelcast.jet.core.DAG used on opensource projects

Example 1 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.

the class WatermarkCoalescer_IntegrationTest method createDag.

private static DAG createDag(Mode mode, List<Object> input1, List<Object> input2) {
    DAG dag = new DAG();
    Vertex mapWmToString = dag.newVertex("mapWmToString", MapWatermarksToString::new).localParallelism(1);
    Vertex sink = dag.newVertex("sink", writeListP("sinkList")).localParallelism(1);
    dag.edge(between(mapWmToString, sink));
    switch(mode) {
        case TWO_EDGES:
            Vertex edge1 = dag.newVertex("edge1", ListSource.supplier(input1)).localParallelism(1);
            Vertex edge2 = dag.newVertex("edge2", ListSource.supplier(input2)).localParallelism(1);
            dag.edge(from(edge1).to(mapWmToString, 0));
            dag.edge(from(edge2).to(mapWmToString, 1));
            break;
        case TWO_QUEUES:
            Vertex edge = dag.newVertex("edge", ListSource.supplier(input1, input2)).localParallelism(2);
            dag.edge(between(edge, mapWmToString));
            break;
        default:
            throw new IllegalArgumentException(mode.toString());
    }
    return dag;
}

Example 2 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.

the class DetermineLocalParallelismTest method testWithParallelism.

private void testWithParallelism(int preferred, int specified, int expected) {
    DAG dag = new DAG();
    dag.newVertex("x", new ValidatingMetaSupplier(preferred, expected)).localParallelism(specified);
    validateExecutionPlans(dag);
}

Example 3 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.

the class ExceptionUtilTest method test_serializationFromNodeToClient.

@Test
public void test_serializationFromNodeToClient() {
    // create one member and one client
    createJetMember();
    JetInstance client = createJetClient();
    RuntimeException exc = new RuntimeException("myException");
    try {
        DAG dag = new DAG();
        dag.newVertex("source", () -> new MockP().setCompleteError(exc)).localParallelism(1);
        client.newJob(dag).join();
    } catch (Exception caught) {
        assertThat(caught.toString(), containsString(exc.toString()));
        TestUtil.assertExceptionInCauses(exc, caught);
    } finally {
        shutdownFactory();
    }
}

Example 4 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.

the class Planner method createDag.

DAG createDag() {
    Map<Transform, List<Transform>> adjacencyMap = pipeline.adjacencyMap();
    validateNoLeakage(adjacencyMap);
    // Calculate greatest common denominator of frame lengths from all transforms in the pipeline
    long frameSizeGcd = Util.gcd(adjacencyMap.keySet().stream().map(Transform::watermarkFrameSize).filter(frameSize -> frameSize > 0).mapToLong(i -> i).toArray());
    WatermarkEmissionPolicy emitPolicy = frameSizeGcd > 0 ? emitByFrame(tumblingWinPolicy(frameSizeGcd)) : noThrottling();
    // Replace emission policy
    for (Transform transform : adjacencyMap.keySet()) {
        if (transform instanceof StreamSourceTransform) {
            StreamSourceTransform t = (StreamSourceTransform) transform;
            if (t.getWmParams() != null) {
                t.setWmGenerationParams(t.getWmParams().withEmitPolicy(emitPolicy));
            }
        } else if (transform instanceof TimestampTransform) {
            TimestampTransform t = (TimestampTransform) transform;
            t.setWmGenerationParams(t.getWmGenParams().withEmitPolicy(emitPolicy));
        }
    }
    Iterable<Transform> sorted = topologicalSort(adjacencyMap, Object::toString);
    for (Transform transform : sorted) {
        transform.addToDag(this);
    }
    return dag;
}

Example 5 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.

the class BackpressureTest method testBackpressure.

@Test
public void testBackpressure() throws Exception {
    DAG dag = new DAG();
    final int member1Port = jet1.getCluster().getLocalMember().getAddress().getPort();
    final Member member2 = jet2.getCluster().getLocalMember();
    final int ptionOwnedByMember2 = jet1.getHazelcastInstance().getPartitionService().getPartitions().stream().filter(p -> p.getOwner().equals(member2)).map(Partition::getPartitionId).findAny().orElseThrow(() -> new RuntimeException("Can't find a partition owned by member " + jet2));
    Vertex source = dag.newVertex("source", ProcessorMetaSupplier.of((Address address) -> ProcessorSupplier.of(address.getPort() == member1Port ? GenerateP::new : noopP())));
    Vertex hiccup = dag.newVertex("hiccup", HiccupP::new);
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeMapP("counts"));
    dag.edge(between(source, hiccup).distributed().partitioned(wholeItem(), (x, y) -> ptionOwnedByMember2)).edge(between(hiccup, sink));
    jet1.newJob(dag).join();
    assertCounts(jet1.getMap("counts"));
}