use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.
the class StreamFilesP_integrationTest method buildDag.
private DAG buildDag() {
DAG dag = new DAG();
Vertex reader = dag.newVertex("reader", streamFilesP(directory.getPath(), UTF_8, "*", false, Util::entry)).localParallelism(1);
Vertex writer = dag.newVertex("writer", writeListP(list.getName())).localParallelism(1);
dag.edge(between(reader, writer));
return dag;
}
use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.
the class StreamFilesP_integrationTest method when_appendingToPreexistingIncompleteLine_then_pickupCompleteLines.
@Test
public void when_appendingToPreexistingIncompleteLine_then_pickupCompleteLines() throws Exception {
DAG dag = buildDag();
// this is a pre-existing file, should not be picked up
File file = createNewFile();
try (PrintWriter writer = new PrintWriter(new FileOutputStream(file, true))) {
// note: no newline appended
writer.write("hello");
}
sleepAtLeastMillis(50);
Future<Void> jobFuture = instance.getJet().newJob(dag).getFuture();
// wait for the processor to initialize
sleepAtLeastSeconds(2);
// pre-existing file should not be picked up
assertEquals(0, list.size());
// this completes the first line and a second one - only the second one should be picked
appendToFile(file, "world", "second line");
// now, all three lines are picked up
assertTrueEventually(() -> assertEquals(1, list.size()));
assertEquals(entry(file.getName(), "second line"), list.get(0));
finishDirectory(jobFuture, file);
}
use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.
the class StreamFilesP_integrationTest method when_newAndModified_then_pickupAddition.
@Test
@Ignore
public void when_newAndModified_then_pickupAddition() throws Exception {
DAG dag = buildDag();
Future<Void> jobFuture = instance.getJet().newJob(dag).getFuture();
// wait for the processor to initialize
sleepAtLeastSeconds(2);
assertEquals(0, list.size());
File file = new File(directory, randomName());
appendToFile(file, "hello", "world");
assertTrueEventually(() -> assertEquals(2, list.size()));
// now add one more line, only this line should be picked up
appendToFile(file, "third line");
assertTrueEventually(() -> assertEquals(3, list.size()));
finishDirectory(jobFuture, file);
}
use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.
the class StreamSocketP_integrationTest method when_dataWrittenToSocket_then_dataImmediatelyEmitted.
@Test
public void when_dataWrittenToSocket_then_dataImmediatelyEmitted() throws Exception {
CountDownLatch latch = new CountDownLatch(1);
// Given
try (ServerSocket socket = new ServerSocket(PORT)) {
spawn(() -> uncheckRun(() -> {
Socket accept1 = socket.accept();
Socket accept2 = socket.accept();
PrintWriter writer1 = new PrintWriter(accept1.getOutputStream());
writer1.write("hello1 \n");
writer1.flush();
PrintWriter writer2 = new PrintWriter(accept2.getOutputStream());
writer2.write("hello2 \n");
writer2.flush();
assertOpenEventually(latch);
writer1.write("world1 \n");
writer1.write("jet1 \n");
writer1.flush();
writer2.write("world2 \n");
writer2.write("jet2 \n");
writer2.flush();
accept1.close();
accept2.close();
}));
DAG dag = new DAG();
Vertex producer = dag.newVertex("producer", streamSocketP(HOST, PORT, UTF_8)).localParallelism(2);
Vertex consumer = dag.newVertex("consumer", writeListP("consumer")).localParallelism(1);
dag.edge(between(producer, consumer));
// When
Job job = instance.getJet().newJob(dag);
IList<Object> list = instance.getList("consumer");
assertTrueEventually(() -> assertEquals(2, list.size()));
latch.countDown();
job.join();
assertEquals(6, list.size());
}
}
use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.
the class StreamSocketP_integrationTest method when_jobCancelled_then_readerClosed.
@Test
public void when_jobCancelled_then_readerClosed() throws Exception {
try (ServerSocket socket = new ServerSocket(PORT)) {
AtomicReference<Socket> accept = new AtomicReference<>();
CountDownLatch acceptationLatch = new CountDownLatch(1);
// Cancellation only works, if there are data on socket. Without data, SocketInputStream.read()
// blocks indefinitely. The StreamSocketP should be improved to use NIO.
spawn(() -> uncheckRun(() -> {
accept.set(socket.accept());
acceptationLatch.countDown();
byte[] word = "jet\n".getBytes();
try (OutputStream outputStream = accept.get().getOutputStream()) {
while (true) {
outputStream.write(word);
outputStream.flush();
Thread.sleep(1000);
}
}
}));
Vertex producer = new Vertex("producer", streamSocketP(HOST, PORT, UTF_8)).localParallelism(1);
Vertex sink = new Vertex("sink", noopP()).localParallelism(1);
DAG dag = new DAG().vertex(producer).vertex(sink).edge(between(producer, sink));
Job job = instance.getJet().newJob(dag);
acceptationLatch.await();
job.cancel();
assertTrueEventually(() -> assertTrue("Socket not closed", accept.get().isClosed()));
}
}
Aggregations