Examples with Vertex com.hazelcast.jet.core.Vertex used on opensource projects

Example 71 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class Processors_globalAggregationIntegrationTest method runTest.

private void runTest(List<Long> sourceItems, Long expectedOutput) throws Exception {
    HazelcastInstance instance = createHazelcastInstance();
    AggregateOperation1<Long, ?, Long> summingOp = summingLong((Long l) -> l);
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", () -> new ListSource(sourceItems)).localParallelism(1);
    Vertex sink = dag.newVertex("sink", writeListP("sink"));
    if (singleStageProcessor) {
        Vertex aggregate = dag.newVertex("aggregate", Processors.aggregateP(summingOp)).localParallelism(1);
        dag.edge(between(source, aggregate).distributed().allToOne("foo")).edge(between(aggregate, sink).isolated());
    } else {
        Vertex accumulate = dag.newVertex("accumulate", Processors.accumulateP(summingOp));
        Vertex combine = dag.newVertex("combine", combineP(summingOp)).localParallelism(1);
        dag.edge(between(source, accumulate)).edge(between(accumulate, combine).distributed().allToOne("foo")).edge(between(combine, sink).isolated());
    }
    instance.getJet().newJob(dag).join();
    IList<Long> sinkList = instance.getList("sink");
    assertEquals(singletonList(expectedOutput), new ArrayList<>(sinkList));
    // wait a little more and make sure, that there are no more frames
    Thread.sleep(1000);
    assertEquals(singletonList(expectedOutput), new ArrayList<>(sinkList));
    assertEquals(expectedOutput, sinkList.get(0));
}

Example 72 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class StreamFilesP_integrationTest method buildDag.

private DAG buildDag() {
    DAG dag = new DAG();
    Vertex reader = dag.newVertex("reader", streamFilesP(directory.getPath(), UTF_8, "*", false, Util::entry)).localParallelism(1);
    Vertex writer = dag.newVertex("writer", writeListP(list.getName())).localParallelism(1);
    dag.edge(between(reader, writer));
    return dag;
}

Example 73 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class StreamSocketP_integrationTest method when_dataWrittenToSocket_then_dataImmediatelyEmitted.

@Test
public void when_dataWrittenToSocket_then_dataImmediatelyEmitted() throws Exception {
    CountDownLatch latch = new CountDownLatch(1);
    // Given
    try (ServerSocket socket = new ServerSocket(PORT)) {
        spawn(() -> uncheckRun(() -> {
            Socket accept1 = socket.accept();
            Socket accept2 = socket.accept();
            PrintWriter writer1 = new PrintWriter(accept1.getOutputStream());
            writer1.write("hello1 \n");
            writer1.flush();
            PrintWriter writer2 = new PrintWriter(accept2.getOutputStream());
            writer2.write("hello2 \n");
            writer2.flush();
            assertOpenEventually(latch);
            writer1.write("world1 \n");
            writer1.write("jet1 \n");
            writer1.flush();
            writer2.write("world2 \n");
            writer2.write("jet2 \n");
            writer2.flush();
            accept1.close();
            accept2.close();
        }));
        DAG dag = new DAG();
        Vertex producer = dag.newVertex("producer", streamSocketP(HOST, PORT, UTF_8)).localParallelism(2);
        Vertex consumer = dag.newVertex("consumer", writeListP("consumer")).localParallelism(1);
        dag.edge(between(producer, consumer));
        // When
        Job job = instance.getJet().newJob(dag);
        IList<Object> list = instance.getList("consumer");
        assertTrueEventually(() -> assertEquals(2, list.size()));
        latch.countDown();
        job.join();
        assertEquals(6, list.size());
    }
}

Example 74 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class StreamSocketP_integrationTest method when_jobCancelled_then_readerClosed.

@Test
public void when_jobCancelled_then_readerClosed() throws Exception {
    try (ServerSocket socket = new ServerSocket(PORT)) {
        AtomicReference<Socket> accept = new AtomicReference<>();
        CountDownLatch acceptationLatch = new CountDownLatch(1);
        // Cancellation only works, if there are data on socket. Without data, SocketInputStream.read()
        // blocks indefinitely. The StreamSocketP should be improved to use NIO.
        spawn(() -> uncheckRun(() -> {
            accept.set(socket.accept());
            acceptationLatch.countDown();
            byte[] word = "jet\n".getBytes();
            try (OutputStream outputStream = accept.get().getOutputStream()) {
                while (true) {
                    outputStream.write(word);
                    outputStream.flush();
                    Thread.sleep(1000);
                }
            }
        }));
        Vertex producer = new Vertex("producer", streamSocketP(HOST, PORT, UTF_8)).localParallelism(1);
        Vertex sink = new Vertex("sink", noopP()).localParallelism(1);
        DAG dag = new DAG().vertex(producer).vertex(sink).edge(between(producer, sink));
        Job job = instance.getJet().newJob(dag);
        acceptationLatch.await();
        job.cancel();
        assertTrueEventually(() -> assertTrue("Socket not closed", accept.get().isClosed()));
    }
}

Example 75 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class WriteBufferedPTest method when_writeBufferedJobFailed_then_bufferDisposed.

@Test
public void when_writeBufferedJobFailed_then_bufferDisposed() throws Exception {
    HazelcastInstance instance = createHazelcastInstance();
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", () -> new NoOutputSourceP());
    Vertex sink = dag.newVertex("sink", getLoggingBufferedWriter()).localParallelism(1);
    dag.edge(Edge.between(source, sink));
    Job job = instance.getJet().newJob(dag);
    // wait for the job to initialize
    Thread.sleep(5000);
    job.cancel();
    assertTrueEventually(() -> assertTrue("No \"dispose\", only: " + events, events.contains("dispose")), 60);
    System.out.println(events);
}