Examples with Pipeline com.hazelcast.jet.pipeline.Pipeline used on opensource projects

Example 46 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class StreamKafkaPTest method when_partitionAddedWhileJobDown_then_consumedFromBeginning.

@Test
public void when_partitionAddedWhileJobDown_then_consumedFromBeginning() throws Exception {
    IList<Entry<Integer, String>> sinkList = instance().getList("sinkList");
    Pipeline p = Pipeline.create();
    Properties properties = properties();
    properties.setProperty("auto.offset.reset", "latest");
    p.readFrom(KafkaSources.<Integer, String>kafka(properties, topic1Name)).withoutTimestamps().writeTo(Sinks.list(sinkList));
    Job job = instance().getJet().newJob(p, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE));
    assertTrueEventually(() -> {
        // This might add multiple `0` events to the topic - we need to do this because the source starts from
        // the latest position and we don't exactly know when it starts, so we try repeatedly
        kafkaTestSupport.produce(topic1Name, 0, "0").get();
        assertFalse(sinkList.isEmpty());
        assertEquals(entry(0, "0"), sinkList.get(0));
    });
    job.suspend();
    assertJobStatusEventually(job, JobStatus.SUSPENDED);
    // Note that the job might not have consumed all the zeroes from the topic at this point
    // When
    kafkaTestSupport.setPartitionCount(topic1Name, INITIAL_PARTITION_COUNT + 2);
    // We produce to a partition that didn't exist during the previous job execution.
    // The job must start reading the new partition from the beginning, otherwise it would miss this item.
    Entry<Integer, String> event = produceEventToNewPartition(INITIAL_PARTITION_COUNT);
    job.resume();
    // All events after the resume will be loaded: the non-consumed zeroes, and the possibly multiple
    // events added in produceEventToNewPartition(). But they must include the event added to the new partition.
    assertTrueEventually(() -> assertThat(sinkList).contains(event));
}

Example 47 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class StreamKafkaPTest method when_autoOffsetResetLatest_then_doesNotReadOldMessages.

@Test
public void when_autoOffsetResetLatest_then_doesNotReadOldMessages() throws Exception {
    IList<Entry<Integer, String>> sinkList = instance().getList("sinkList");
    Pipeline p = Pipeline.create();
    Properties properties = properties();
    properties.setProperty("auto.offset.reset", "latest");
    p.readFrom(KafkaSources.<Integer, String>kafka(properties, topic1Name)).withoutTimestamps().writeTo(Sinks.list(sinkList));
    kafkaTestSupport.produce(topic1Name, 0, "0").get();
    instance().getJet().newJob(p);
    assertTrueAllTheTime(() -> assertTrue(sinkList.isEmpty()), 2);
}

Example 48 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class StreamKafkaPTest method integrationTest.

private void integrationTest(ProcessingGuarantee guarantee) throws Exception {
    int messageCount = 20;
    HazelcastInstance[] instances = new HazelcastInstance[2];
    Arrays.setAll(instances, i -> createHazelcastInstance());
    Pipeline p = Pipeline.create();
    p.readFrom(KafkaSources.kafka(properties(), topic1Name, topic2Name)).withoutTimestamps().writeTo(Sinks.list("sink"));
    JobConfig config = new JobConfig();
    config.setProcessingGuarantee(guarantee);
    config.setSnapshotIntervalMillis(500);
    Job job = instances[0].getJet().newJob(p, config);
    sleepSeconds(3);
    for (int i = 0; i < messageCount; i++) {
        kafkaTestSupport.produce(topic1Name, i, Integer.toString(i));
        kafkaTestSupport.produce(topic2Name, i - messageCount, Integer.toString(i - messageCount));
    }
    IList<Object> list = instances[0].getList("sink");
    assertTrueEventually(() -> {
        assertEquals(messageCount * 2, list.size());
        for (int i = 0; i < messageCount; i++) {
            Entry<Integer, String> entry1 = createEntry(i);
            Entry<Integer, String> entry2 = createEntry(i - messageCount);
            assertTrue("missing entry: " + entry1, list.contains(entry1));
            assertTrue("missing entry: " + entry2, list.contains(entry2));
        }
    }, 15);
    if (guarantee != ProcessingGuarantee.NONE) {
        // wait until a new snapshot appears
        JobRepository jr = new JobRepository(instances[0]);
        long currentMax = jr.getJobExecutionRecord(job.getId()).snapshotId();
        assertTrueEventually(() -> {
            JobExecutionRecord jobExecutionRecord = jr.getJobExecutionRecord(job.getId());
            assertNotNull("jobExecutionRecord == null", jobExecutionRecord);
            long newMax = jobExecutionRecord.snapshotId();
            assertTrue("no snapshot produced", newMax > currentMax);
            System.out.println("snapshot " + newMax + " found, previous was " + currentMax);
        });
        // Bring down one member. Job should restart and drain additional items (and maybe
        // some of the previous duplicately).
        instances[1].getLifecycleService().terminate();
        Thread.sleep(500);
        for (int i = messageCount; i < 2 * messageCount; i++) {
            kafkaTestSupport.produce(topic1Name, i, Integer.toString(i));
            kafkaTestSupport.produce(topic2Name, i - messageCount, Integer.toString(i - messageCount));
        }
        assertTrueEventually(() -> {
            assertTrue("Not all messages were received", list.size() >= messageCount * 4);
            for (int i = 0; i < 2 * messageCount; i++) {
                Entry<Integer, String> entry1 = createEntry(i);
                Entry<Integer, String> entry2 = createEntry(i - messageCount);
                assertTrue("missing entry: " + entry1.toString(), list.contains(entry1));
                assertTrue("missing entry: " + entry2.toString(), list.contains(entry2));
            }
        }, 10);
    }
    assertFalse(job.getFuture().isDone());
    // cancel the job
    job.cancel();
    assertTrueEventually(() -> assertTrue(job.getFuture().isDone()));
}

Example 49 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class WriteKafkaPTest method testWriteToSpecificPartitions.

@Test
public void testWriteToSpecificPartitions() {
    String localTopic = topic;
    Pipeline p = Pipeline.create();
    p.readFrom(Sources.map(sourceIMap)).writeTo(KafkaSinks.kafka(properties, e -> new ProducerRecord<>(localTopic, e.getKey(), e.getKey(), e.getValue())));
    instance().getJet().newJob(p).join();
    kafkaTestSupport.assertTopicContentsEventually(topic, sourceIMap, true);
}

Example 50 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class WriteKafkaPTest method testWriteToTopic.

@Test
public void testWriteToTopic() {
    Pipeline p = Pipeline.create();
    p.readFrom(Sources.map(sourceIMap)).writeTo(KafkaSinks.kafka(properties, topic));
    instance().getJet().newJob(p).join();
    kafkaTestSupport.assertTopicContentsEventually(topic, sourceIMap, false);
}