Examples with DAG com.hazelcast.jet.core.DAG used on opensource projects

Example 21 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.

the class StreamFilesP_integrationTest method when_newAndModified_then_pickupAddition.

@Test
@Ignore
public void when_newAndModified_then_pickupAddition() throws Exception {
    DAG dag = buildDag();
    Future<Void> jobFuture = instance.newJob(dag).getFuture();
    // wait for the processor to initialize
    sleepAtLeastSeconds(2);
    assertEquals(0, list.size());
    File file = new File(directory, randomName());
    appendToFile(file, "hello", "world");
    assertTrueEventually(() -> assertEquals(2, list.size()));
    // now add one more line, only this line should be picked up
    appendToFile(file, "third line");
    assertTrueEventually(() -> assertEquals(3, list.size()));
    finishDirectory(jobFuture, file);
}

Example 22 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.

the class StreamFilesP_integrationTest method when_appendingToPreexistingIncompleteLine_then_pickupCompleteLines.

@Test
public void when_appendingToPreexistingIncompleteLine_then_pickupCompleteLines() throws Exception {
    DAG dag = buildDag();
    // this is a pre-existing file, should not be picked up
    File file = createNewFile();
    try (PrintWriter writer = new PrintWriter(new FileOutputStream(file, true))) {
        // note: no newline appended
        writer.write("hello");
    }
    sleepAtLeastMillis(50);
    Future<Void> jobFuture = instance.newJob(dag).getFuture();
    // wait for the processor to initialize
    sleepAtLeastSeconds(2);
    // pre-existing file should not be picked up
    assertEquals(0, list.size());
    // this completes the first line and a second one - only the second one should be picked
    appendToFile(file, "world", "second line");
    // now, all three lines are picked up
    assertTrueEventually(() -> assertEquals(1, list.size()));
    assertEquals(entry(file.getName(), "second line"), list.get(0));
    finishDirectory(jobFuture, file);
}

Example 23 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.

the class StreamFilesP_integrationTest method when_fileWithManyLines_then_emitCooperatively.

@Test
public void when_fileWithManyLines_then_emitCooperatively() throws Exception {
    DAG dag = buildDag();
    Future<Void> jobFuture = instance.newJob(dag).getFuture();
    // wait for the processor to initialize
    sleepAtLeastSeconds(2);
    assertEquals(0, list.size());
    File subdir = new File(directory, "subdir");
    assertTrue(subdir.mkdir());
    File fileInSubdir = new File(subdir, randomName());
    appendToFile(fileInSubdir, IntStream.range(0, 5000).mapToObj(String::valueOf).toArray(String[]::new));
    // move the file to watchedDirectory
    File file = new File(directory, fileInSubdir.getName());
    assertTrue(fileInSubdir.renameTo(file));
    assertTrueEventually(() -> assertEquals(5000, list.size()));
    finishDirectory(jobFuture, file, subdir);
}

Example 24 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.

the class StreamSocketP_integrationTest method when_dataWrittenToSocket_then_dataImmediatelyEmitted.

@Test
public void when_dataWrittenToSocket_then_dataImmediatelyEmitted() throws Exception {
    CountDownLatch latch = new CountDownLatch(1);
    // Given
    try (ServerSocket socket = new ServerSocket(PORT)) {
        new Thread(() -> uncheckRun(() -> {
            Socket accept1 = socket.accept();
            Socket accept2 = socket.accept();
            PrintWriter writer1 = new PrintWriter(accept1.getOutputStream());
            writer1.write("hello1 \n");
            writer1.flush();
            PrintWriter writer2 = new PrintWriter(accept2.getOutputStream());
            writer2.write("hello2 \n");
            writer2.flush();
            assertOpenEventually(latch);
            writer1.write("world1 \n");
            writer1.write("jet1 \n");
            writer1.flush();
            writer2.write("world2 \n");
            writer2.write("jet2 \n");
            writer2.flush();
            accept1.close();
            accept2.close();
        })).start();
        DAG dag = new DAG();
        Vertex producer = dag.newVertex("producer", streamSocketP(HOST, PORT, UTF_8)).localParallelism(2);
        Vertex consumer = dag.newVertex("consumer", writeListP("consumer")).localParallelism(1);
        dag.edge(between(producer, consumer));
        // When
        Job job = instance.newJob(dag);
        IList<Object> list = instance.getList("consumer");
        assertTrueEventually(() -> assertEquals(2, list.size()));
        latch.countDown();
        job.join();
        assertEquals(6, list.size());
    }
}

Example 25 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.

the class WriteFilePTest method when_slowSource_then_fileFlushedAfterEachItem.

@Test
public void when_slowSource_then_fileFlushedAfterEachItem() {
    // Given
    int numItems = 10;
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", () -> new SlowSourceP(semaphore, numItems)).localParallelism(1);
    Vertex sink = dag.newVertex("sink", writeFileP(directory.toString(), Object::toString, StandardCharsets.UTF_8, false)).localParallelism(1);
    dag.edge(between(source, sink));
    Job job = instance.newJob(dag);
    // wait, until the file is created
    assertTrueEventually(() -> assertTrue(Files.exists(file)));
    for (int i = 0; i < numItems; i++) {
        // When
        semaphore.release();
        int finalI = i;
        // Then
        assertTrueEventually(() -> checkFileContents(StandardCharsets.UTF_8, finalI + 1), 5);
    }
    // wait for the job to finish
    job.join();
}