Examples with JobConfig com.hazelcast.jet.config.JobConfig used on opensource projects

Example 1 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast-jet by hazelcast.

the class StreamKafkaPTest method integrationTest.

private void integrationTest(ProcessingGuarantee guarantee) throws Exception {
    int messageCount = 20;
    JetInstance[] instances = new JetInstance[2];
    Arrays.setAll(instances, i -> createJetMember());
    Pipeline p = Pipeline.create();
    p.drawFrom(KafkaSources.kafka(properties, topic1Name, topic2Name)).drainTo(Sinks.list("sink"));
    JobConfig config = new JobConfig();
    config.setProcessingGuarantee(guarantee);
    config.setSnapshotIntervalMillis(500);
    Job job = instances[0].newJob(p, config);
    sleepAtLeastSeconds(3);
    for (int i = 0; i < messageCount; i++) {
        produce(topic1Name, i, Integer.toString(i));
        produce(topic2Name, i - messageCount, Integer.toString(i - messageCount));
    }
    IList<Object> list = instances[0].getList("sink");
    assertTrueEventually(() -> {
        assertEquals(messageCount * 2, list.size());
        for (int i = 0; i < messageCount; i++) {
            Entry<Integer, String> entry1 = createEntry(i);
            Entry<Integer, String> entry2 = createEntry(i - messageCount);
            assertTrue("missing entry: " + entry1, list.contains(entry1));
            assertTrue("missing entry: " + entry2, list.contains(entry2));
        }
    }, 5);
    if (guarantee != ProcessingGuarantee.NONE) {
        // wait until the items are consumed and a new snapshot appears
        assertTrueEventually(() -> assertTrue(list.size() == messageCount * 2));
        IMapJet<Long, Object> snapshotsMap = instances[0].getMap(SnapshotRepository.snapshotsMapName(job.getId()));
        Long currentMax = maxSuccessfulSnapshot(snapshotsMap);
        assertTrueEventually(() -> {
            Long newMax = maxSuccessfulSnapshot(snapshotsMap);
            assertTrue("no snapshot produced", newMax != null && !newMax.equals(currentMax));
            System.out.println("snapshot " + newMax + " found, previous was " + currentMax);
        });
        // Bring down one member. Job should restart and drain additional items (and maybe
        // some of the previous duplicately).
        instances[1].shutdown();
        Thread.sleep(500);
        for (int i = messageCount; i < 2 * messageCount; i++) {
            produce(topic1Name, i, Integer.toString(i));
            produce(topic2Name, i - messageCount, Integer.toString(i - messageCount));
        }
        assertTrueEventually(() -> {
            assertTrue("Not all messages were received", list.size() >= messageCount * 4);
            for (int i = 0; i < 2 * messageCount; i++) {
                Entry<Integer, String> entry1 = createEntry(i);
                Entry<Integer, String> entry2 = createEntry(i - messageCount);
                assertTrue("missing entry: " + entry1.toString(), list.contains(entry1));
                assertTrue("missing entry: " + entry2.toString(), list.contains(entry2));
            }
        }, 10);
    }
    assertFalse(job.getFuture().isDone());
    // cancel the job
    job.cancel();
    assertTrueEventually(() -> assertTrue(job.getFuture().isDone()));
}

Example 2 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast-jet by hazelcast.

the class WatermarkCoalescer_IntegrationTest method when_i1_activeNoWm_i2_activeNoWm_then_wmForwardedAfterDelay.

@Test
public void when_i1_activeNoWm_i2_activeNoWm_then_wmForwardedAfterDelay() {
    dag = createDag(mode, singletonList(wm(100)), singletonList(wm(150)));
    JobConfig config = new JobConfig().setMaxWatermarkRetainMillis(5000);
    long time = System.nanoTime();
    instance.newJob(dag, config);
    assertTrueEventually(() -> assertEquals(1, sinkList.size()), 3);
    assertEquals("wm(100)", sinkList.get(0));
    assertTrueEventually(() -> assertEquals(2, sinkList.size()), 6);
    assertEquals("wm(150)", sinkList.get(1));
    long elapsedMs = NANOSECONDS.toMillis(System.nanoTime() - time);
    assertTrue("Too little elapsed time, WM probably emitted immediately: " + elapsedMs, elapsedMs > 3000);
}

Example 3 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast-jet by hazelcast.

the class WatermarkCoalescer_IntegrationTest method when_i1_active_i2_active_then_wmForwardedImmediately.

@Test
public void when_i1_active_i2_active_then_wmForwardedImmediately() {
    dag = createDag(mode, singletonList(wm(100)), singletonList(wm(100)));
    JobConfig config = new JobConfig().setMaxWatermarkRetainMillis(5000);
    instance.newJob(dag, config);
    assertTrueEventually(() -> assertEquals(1, sinkList.size()), 3);
    assertEquals("wm(100)", sinkList.get(0));
}

Example 4 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast-jet by hazelcast.

the class WatermarkCoalescer_IntegrationTest method when_i1_idle_i2_idle_then_idleMessageForwardedImmediately.

@Test
public void when_i1_idle_i2_idle_then_idleMessageForwardedImmediately() {
    dag = createDag(mode, singletonList(IDLE_MESSAGE), singletonList(IDLE_MESSAGE));
    JobConfig config = new JobConfig().setMaxWatermarkRetainMillis(5000);
    instance.newJob(dag, config);
    // the idle message should not be presented to the processor
    assertTrueAllTheTime(() -> assertEquals(0, sinkList.size()), 3);
}

Example 5 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast-jet by hazelcast.

the class WatermarkCoalescer_IntegrationTest method when_i1_activeNoWm_i2_aheadAndIdle_then_wmForwardedAfterADelay.

@Test
public void when_i1_activeNoWm_i2_aheadAndIdle_then_wmForwardedAfterADelay() {
    dag = createDag(mode, singletonList(entry(1, 1)), asList(wm(150), IDLE_MESSAGE));
    JobConfig config = new JobConfig().setMaxWatermarkRetainMillis(5000);
    instance.newJob(dag, config);
    assertTrueEventually(() -> assertEquals(1, sinkList.size()), 3);
    assertEquals(entry(1, 1), sinkList.get(0));
    long time = System.nanoTime();
    assertTrueEventually(() -> assertEquals(2, sinkList.size()), 6);
    assertEquals("wm(150)", sinkList.get(1));
    long elapsedMs = NANOSECONDS.toMillis(System.nanoTime() - time);
    assertTrue("Too little elapsed time, WM probably emitted immediately", elapsedMs > 3000);
}