Examples with DAG com.hazelcast.jet.core.DAG used on opensource projects

Example 71 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class WriteBufferedPTest method when_writeBufferedJobFailed_then_bufferDisposed.

@Test
public void when_writeBufferedJobFailed_then_bufferDisposed() throws Exception {
    HazelcastInstance instance = createHazelcastInstance();
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", () -> new NoOutputSourceP());
    Vertex sink = dag.newVertex("sink", getLoggingBufferedWriter()).localParallelism(1);
    dag.edge(Edge.between(source, sink));
    Job job = instance.getJet().newJob(dag);
    // wait for the job to initialize
    Thread.sleep(5000);
    job.cancel();
    assertTrueEventually(() -> assertTrue("No \"dispose\", only: " + events, events.contains("dispose")), 60);
    System.out.println(events);
}

Example 72 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_peekAndRebalanceAndMap_then_dagEdgeDistributed.

@Test
public void when_peekAndRebalanceAndMap_then_dagEdgeDistributed() {
    // Given
    List<Integer> input = sequence(itemCount);
    BatchStage<Integer> srcStage = batchStageFromList(input);
    FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
    // When
    BatchStage<String> mapped = srcStage.peek().rebalance().map(formatFn);
    // Then
    mapped.writeTo(sink);
    DAG dag = p.toDag();
    Edge srcToMap = dag.getInboundEdges("map").get(0);
    assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", srcToMap.getPartitioner());
    execute();
    assertEquals(streamToString(input.stream(), formatFn), streamToString(sinkStreamOf(String.class), identity()));
}

Example 73 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class ProcessorTransformParallelismTest method when_nonCooperative_defaultLP_then_UsesProvidedLP.

@Test
public void when_nonCooperative_defaultLP_then_UsesProvidedLP() {
    // When
    DAG dag = applyTransformAndGetDag(nonCooperative_defaultLP);
    // Then
    Vertex tsVertex = dag.getVertex(transformName);
    assertEquals(LOCAL_PARALLELISM, tsVertex.determineLocalParallelism(DEFAULT_PARALLELISM));
}

Example 74 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class RebalanceStreamStageTest method when_peekAndRebalanceByKeyAndMap_then_dagEdgePartitionedDistributed.

@Test
public void when_peekAndRebalanceByKeyAndMap_then_dagEdgePartitionedDistributed() {
    // Given
    List<Integer> input = sequence(itemCount);
    StreamStage<Integer> srcStage = streamStageFromList(input);
    FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
    // When
    StreamStage<String> mapped = srcStage.peek().rebalance(i -> i).map(formatFn);
    // Then
    mapped.writeTo(sink);
    DAG dag = p.toDag();
    Edge srcToMap = dag.getInboundEdges("map").get(0);
    assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
    assertNotNull("Rebalanced by key, the edge must be partitioned", srcToMap.getPartitioner());
}

Example 75 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class RebalanceStreamStageTest method when_rebalanceByKeyAndMap_then_dagEdgePartitionedDistributed.

@Test
public void when_rebalanceByKeyAndMap_then_dagEdgePartitionedDistributed() {
    // Given
    List<Integer> input = sequence(itemCount);
    StreamStage<Integer> srcStage = streamStageFromList(input);
    FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
    // When
    StreamStage<String> mapped = srcStage.rebalance(i -> i).map(formatFn);
    // Then
    mapped.writeTo(sink);
    DAG dag = p.toDag();
    Edge srcToMap = dag.getInboundEdges("map").get(0);
    assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
    assertNotNull("Rebalanced by key, the edge must be partitioned", srcToMap.getPartitioner());
}