Examples with DAG com.hazelcast.jet.core.DAG used on opensource projects

Example 16 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.

the class HazelcastRemoteConnectorTest method when_writeRemoteCache.

@Test
public void when_writeRemoteCache() {
    populateCache(jet.getCacheManager().getCache(SOURCE_NAME));
    DAG dag = new DAG();
    Vertex producer = dag.newVertex(SOURCE_NAME, readCacheP(SOURCE_NAME));
    Vertex consumer = dag.newVertex(SINK_NAME, writeRemoteCacheP(SINK_NAME, clientConfig));
    dag.edge(between(producer, consumer));
    executeAndWait(dag);
    assertEquals(ITEM_COUNT, hz.getCacheManager().getCache(SINK_NAME).size());
}

Example 17 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.

the class ReadFilesPTest method when_directory_then_ignore.

@Test
public void when_directory_then_ignore() {
    DAG dag = buildDag(null);
    File file1 = new File(directory, randomName());
    assertTrue(file1.mkdir());
    instance.newJob(dag).join();
    assertEquals(0, list.size());
    finishDirectory(file1);
}

Example 18 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.

the class ReadFilesPTest method when_glob_the_useGlob.

@Test
public void when_glob_the_useGlob() throws Exception {
    DAG dag = buildDag("file2.*");
    File file1 = new File(directory, "file1.txt");
    appendToFile(file1, "hello", "world");
    File file2 = new File(directory, "file2.txt");
    appendToFile(file2, "hello2", "world2");
    instance.newJob(dag).join();
    assertEquals(Arrays.asList(entry("file2.txt", "hello2"), entry("file2.txt", "world2")), new ArrayList<>(list));
    finishDirectory(file1, file2);
}

Example 19 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.

the class ReadFilesPTest method buildDag.

private DAG buildDag(String glob) {
    if (glob == null) {
        glob = "*";
    }
    DAG dag = new DAG();
    Vertex reader = dag.newVertex("reader", readFilesP(directory.getPath(), StandardCharsets.UTF_8, glob, Util::entry)).localParallelism(1);
    Vertex writer = dag.newVertex("writer", writeListP(list.getName())).localParallelism(1);
    dag.edge(between(reader, writer));
    return dag;
}

Example 20 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.

the class StreamFilesP_integrationTest method when_withCrlf_then_pickupCompleteLines.

@Test
public void when_withCrlf_then_pickupCompleteLines() throws Exception {
    DAG dag = buildDag();
    // this is a pre-existing file, should not be picked up
    File file = createNewFile();
    try (PrintWriter writer = new PrintWriter(new FileOutputStream(file, true))) {
        // note: no newline appended
        writer.write("hello world\r");
    }
    sleepAtLeastMillis(50);
    Future<Void> jobFuture = instance.newJob(dag).getFuture();
    // wait for the processor to initialize
    sleepAtLeastSeconds(2);
    // pre-existing file should not be picked up
    assertEquals(0, list.size());
    // this completes the first line and a second one - only the second one should be picked
    try (PrintWriter writer = new PrintWriter(new FileOutputStream(file, true))) {
        // note: no newline appended
        writer.write("\nsecond line\r\n");
    }
    // now, all three lines are picked up
    assertTrueEventually(() -> assertEquals(1, list.size()));
    assertEquals(entry(file.getName(), "second line"), list.get(0));
    finishDirectory(jobFuture, file);
}