Examples with Pipeline com.hazelcast.jet.pipeline.Pipeline used on opensource projects

Example 11 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast-jet by hazelcast.

the class WindowGroupTransform_IntegrationTest method testWindowDefinition.

@Test
public void testWindowDefinition() {
    Pipeline p = Pipeline.create();
    SlidingWindowDef tumbling = WindowDefinition.tumbling(2);
    StageWithGroupingAndWindow<Entry<Long, String>, Character> stage = p.drawFrom(Sources.<Long, String>mapJournal("source", START_FROM_OLDEST)).groupingKey(entry -> entry.getValue().charAt(0)).window(tumbling);
    assertEquals(tumbling, stage.windowDefinition());
}

Example 12 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast-jet by hazelcast.

the class WindowGroupTransform_IntegrationTest method testSliding_windowFirst_aggregate3.

@Test
public void testSliding_windowFirst_aggregate3() {
    IMap<Long, String> map = instance.getMap("source");
    // key is timestamp
    map.put(0L, "foo");
    map.put(2L, "taz");
    map.put(10L, "flush-item");
    IMap<Long, String> map1 = instance.getMap("source1");
    // key is timestamp
    map1.put(0L, "faa");
    map1.put(2L, "tuu");
    map1.put(10L, "flush-item");
    IMap<Long, String> map2 = instance.getMap("source2");
    // key is timestamp
    map2.put(0L, "fzz");
    map2.put(2L, "tcc");
    map2.put(10L, "flush-item");
    Pipeline p = Pipeline.create();
    StreamStageWithGrouping<Entry<Long, String>, Character> stage1 = p.drawFrom(Sources.<Long, String>mapJournal("source1", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).groupingKey(entry -> entry.getValue().charAt(0));
    StreamStageWithGrouping<Entry<Long, String>, Character> stage2 = p.drawFrom(Sources.<Long, String>mapJournal("source2", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).groupingKey(entry -> entry.getValue().charAt(0));
    p.drawFrom(Sources.<Long, String>mapJournal("source", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).window(WindowDefinition.tumbling(2)).groupingKey(entry -> entry.getValue().charAt(0)).aggregate3(stage1, stage2, toThreeBags()).peek().drainTo(Sinks.list("sink"));
    instance.newJob(p);
    assertTrueEventually(() -> {
        assertEquals(listToString(asList(new TimestampedEntry<>(2, 'f', ThreeBags.threeBags(asList(entry(0L, "foo")), asList(entry(0L, "faa")), asList(entry(0L, "fzz")))), new TimestampedEntry<>(4, 't', ThreeBags.threeBags(asList(entry(2L, "taz")), asList(entry(2L, "tuu")), asList(entry(2L, "tcc")))))), listToString(instance.getHazelcastInstance().getList("sink")));
    }, 5);
}

Example 13 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast-jet by hazelcast.

the class WindowGroupTransform_IntegrationTest method testSession_groupingFirst.

@Test
public void testSession_groupingFirst() {
    Pipeline p = Pipeline.create();
    p.drawFrom(Sources.<Long, String>mapJournal("source", JournalInitialPosition.START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).groupingKey(entry -> entry.getValue().charAt(0)).window(WindowDefinition.session(2)).aggregate(toSet(), WindowResult::new).drainTo(Sinks.list("sink"));
    testSession(p);
}

Example 14 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast-jet by hazelcast.

the class WindowGroupTransform_IntegrationTest method testSliding_groupingFirst_withNonStreamingSource.

@Test
public void testSliding_groupingFirst_withNonStreamingSource() {
    IList<Entry<Long, String>> list = instance.getList("source");
    list.add(entry(0L, "foo"));
    list.add(entry(1L, "bar"));
    list.add(entry(2L, "baz"));
    list.add(entry(3L, "booze"));
    Pipeline p = Pipeline.create();
    p.drawFrom(Sources.<Entry<Long, String>>list("source")).addTimestamps(Entry::getKey, 0).groupingKey(entry -> entry.getValue().charAt(0)).window(WindowDefinition.tumbling(2)).aggregate(toSet()).drainTo(Sinks.list("sink"));
    instance.newJob(p).join();
    assertTrueEventually(() -> {
        assertEquals(set(new TimestampedEntry<>(2, 'f', set(entry(0L, "foo"))), new TimestampedEntry<>(2, 'b', set(entry(1L, "bar"))), new TimestampedEntry<>(4, 'b', set(entry(2L, "baz"), entry(3L, "booze")))), new HashSet<>(instance.getHazelcastInstance().getList("sink")));
    }, 5);
}

Example 15 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast-jet by hazelcast.

the class WriteFilePTest method smokeTest_smallFile.

@Test
public void smokeTest_smallFile() throws Exception {
    // Given
    Pipeline p = buildPipeline(null, null, false);
    addItemsToList(0, 10);
    // When
    instance.newJob(p).join();
    // Then
    checkFileContents(StandardCharsets.UTF_8, 10);
}