Examples with Pipeline com.hazelcast.jet.pipeline.Pipeline used on opensource projects

Example 51 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class WriteKafkaPTest method when_recordLingerEnabled_then_sentOnCompletion.

@Test
public void when_recordLingerEnabled_then_sentOnCompletion() {
    // When
    // 1 hour
    properties.setProperty("linger.ms", "3600000");
    // Given
    Pipeline p = Pipeline.create();
    p.readFrom(Sources.<Entry<String, String>>batchFromProcessor("source", ProcessorMetaSupplier.of(ProcessorWithEntryAndLatch::new))).writeTo(KafkaSinks.kafka(properties, topic));
    Job job = instance().getJet().newJob(p);
    // the event should not appear in the topic due to linger.ms
    try (KafkaConsumer<Integer, String> consumer = kafkaTestSupport.createConsumer(topic)) {
        assertTrueAllTheTime(() -> assertEquals(0, consumer.poll(Duration.ofMillis(100)).count()), 2);
    }
    // Then
    ProcessorWithEntryAndLatch.isDone = true;
    job.join();
    logger.info("Job finished");
    kafkaTestSupport.assertTopicContentsEventually(topic, singletonMap(0, "v"), false);
}

Example 52 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class Hz3SourcesTest method readFromMapManyItems.

@Test
public void readFromMapManyItems() {
    IMap<Integer, String> map = hz3.getMap("test-map");
    Map<Integer, String> items = new HashMap<>();
    for (int i = 0; i < 10_000; i++) {
        items.put(i, "item " + i);
    }
    map.putAll(items);
    HazelcastInstance hz = createHazelcastInstance();
    Pipeline p = Pipeline.create();
    BatchSource<Map.Entry<Integer, String>> source = Hz3Sources.remoteMap("test-map", HZ3_CLIENT_CONFIG);
    p.readFrom(source).writeTo(Sinks.map("test-result"));
    JobConfig config = getJobConfig(source.name());
    Job job = hz.getJet().newJob(p, config);
    job.join();
    IMap<Integer, String> result = hz.getMap("test-result");
    assertThat(result.entrySet()).isEqualTo(items.entrySet());
}

Example 53 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class Hz3SourcesTest method when_readFromInstanceDown_then_shouldThrowJetException.

@Test
public void when_readFromInstanceDown_then_shouldThrowJetException() {
    HazelcastInstance hz = createHazelcastInstance();
    Pipeline p = Pipeline.create();
    BatchSource<Map.Entry<Integer, String>> source = Hz3Sources.remoteMap("test-map", HZ3_DOWN_CLIENT_CONFIG);
    p.readFrom(source).map(Map.Entry::getValue).writeTo(Sinks.list("test-result"));
    JobConfig config = getJobConfig(source.name());
    Job job = hz.getJet().newJob(p, config);
    assertThatThrownBy(() -> job.join()).hasStackTraceContaining(JetException.class.getName()).hasStackTraceContaining("Unable to connect to any cluster");
}

Example 54 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class Hz3SourcesTest method readFromEmptyMap.

@Test
public void readFromEmptyMap() {
    // make sure the map exists
    hz3.getMap("test-map");
    HazelcastInstance hz = createHazelcastInstance();
    Pipeline p = Pipeline.create();
    BatchSource<Map.Entry<Integer, String>> source = Hz3Sources.remoteMap("test-map", HZ3_CLIENT_CONFIG);
    p.readFrom(source).map(Map.Entry::getValue).writeTo(Sinks.list("test-result"));
    JobConfig config = getJobConfig(source.name());
    Job job = hz.getJet().newJob(p, config);
    job.join();
    IList<String> result = hz.getList("test-result");
    assertThat(result).isEmpty();
}

Example 55 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class FileNotFoundReadHadoopPTest method shouldNotFailForNonExistingPath.

@Test
public void shouldNotFailForNonExistingPath() throws IOException {
    paths.add(new org.apache.hadoop.fs.Path(directory + File.separator + "not-exists"));
    Pipeline p = Pipeline.create();
    p.readFrom(HadoopSources.inputFormat(configuration(true), (k, v) -> v.toString())).setLocalParallelism(1).writeTo(Sinks.logger());
    // Should just pass, no need to check results as there are none
    instance().getJet().newJob(p).join();
}