Examples with DAG com.hazelcast.jet.core.DAG used on opensource projects

Example 56 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class HazelcastRemoteConnectorTest method when_readRemoteMap_withPredicateAndFunction.

@Test
public void when_readRemoteMap_withPredicateAndFunction() {
    populateMap(remoteHz.getMap(SOURCE_NAME));
    DAG dag = new DAG();
    Vertex source = dag.newVertex(SOURCE_NAME, readRemoteMapP(SOURCE_NAME, clientConfig, e -> !e.getKey().equals(0), Entry::getValue)).localParallelism(4);
    Vertex sink = dag.newVertex(SINK_NAME, writeListP(SINK_NAME)).localParallelism(1);
    dag.edge(between(source, sink));
    executeAndWait(dag);
    IList<Object> list = localHz.getList(SINK_NAME);
    assertEquals(ITEM_COUNT - 1, list.size());
    assertFalse(list.contains(0));
    assertTrue(list.contains(1));
}

Example 57 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class HazelcastRemoteConnectorTest method when_readRemoteCache.

@Test
public void when_readRemoteCache() {
    populateCache(remoteHz.getCacheManager().getCache(SOURCE_NAME));
    DAG dag = new DAG();
    Vertex source = dag.newVertex(SOURCE_NAME, readRemoteCacheP(SOURCE_NAME, clientConfig));
    Vertex sink = dag.newVertex(SINK_NAME, writeListP(SINK_NAME));
    dag.edge(between(source, sink));
    executeAndWait(dag);
    assertEquals(ITEM_COUNT, localHz.getList(SINK_NAME).size());
}

Example 58 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class HazelcastRemoteConnectorTest method when_streamRemoteCache_withPredicateAndProjection.

@Test
public void when_streamRemoteCache_withPredicateAndProjection() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex(SOURCE_NAME, SourceProcessors.<Integer, Integer, Integer>streamRemoteCacheP(SOURCE_NAME, clientConfig, event -> !event.getKey().equals(0), EventJournalCacheEvent::getKey, START_FROM_OLDEST, eventTimePolicy(i -> i, limitingLag(0), 1, 0, 10_000)));
    Vertex sink = dag.newVertex(SINK_NAME, writeListP(SINK_NAME));
    dag.edge(between(source, sink));
    Job job = localHz.getJet().newJob(dag);
    populateCache(remoteHz.getCacheManager().getCache(SOURCE_NAME));
    assertSizeEventually(ITEM_COUNT - 1, localHz.getList(SINK_NAME));
    assertFalse(localHz.getList(SINK_NAME).contains(0));
    assertTrue(localHz.getList(SINK_NAME).contains(1));
    job.cancel();
}

Example 59 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class HazelcastRemoteConnectorTest method when_streamRemoteCache.

@Test
public void when_streamRemoteCache() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex(SOURCE_NAME, streamRemoteCacheP(SOURCE_NAME, clientConfig, START_FROM_OLDEST, eventTimePolicy(Entry<Integer, Integer>::getValue, limitingLag(0), 1, 0, 10_000))).localParallelism(4);
    Vertex sink = dag.newVertex(SINK_NAME, writeListP(SINK_NAME)).localParallelism(1);
    dag.edge(between(source, sink));
    Job job = localHz.getJet().newJob(dag);
    populateCache(remoteHz.getCacheManager().getCache(SOURCE_NAME));
    assertSizeEventually(ITEM_COUNT, localHz.getList(SINK_NAME));
    job.cancel();
}

Example 60 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class RebalanceStreamStageTest method when_rebalanceAndMap_then_dagEdgeDistributed.

@Test
public void when_rebalanceAndMap_then_dagEdgeDistributed() {
    // Given
    List<Integer> input = sequence(itemCount);
    StreamStage<Integer> srcStage = streamStageFromList(input);
    FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
    // When
    StreamStage<String> mapped = srcStage.rebalance().map(formatFn);
    // Then
    mapped.writeTo(sink);
    DAG dag = p.toDag();
    Edge srcToMap = dag.getInboundEdges("map").get(0);
    assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", srcToMap.getPartitioner());
    execute();
    assertEquals(streamToString(input.stream(), formatFn), streamToString(sinkStreamOf(String.class), identity()));
}