Examples with Pipeline com.hazelcast.jet.pipeline.Pipeline used on opensource projects

Example 6 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast-jet by hazelcast.

the class GroupTransform_IntegrationTest method test.

@Test
public void test() {
    IMap<Long, String> map = instance.getMap("source");
    map.put(0L, "foo");
    map.put(1L, "bar");
    map.put(2L, "baz");
    Pipeline p = Pipeline.create();
    p.drawFrom(Sources.<Long, String>map("source")).groupingKey(entry -> entry.getValue().charAt(0)).aggregate(toSet()).drainTo(Sinks.list("sink"));
    instance.newJob(p).join();
    assertEquals(set(entry('f', set(entry(0L, "foo"))), entry('b', set(entry(1L, "bar"), entry(2L, "baz")))), new HashSet<>(instance.getHazelcastInstance().getList("sink")));
}

Example 7 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast-jet by hazelcast.

the class WindowAggregateTransform_IntegrationTest method test_aggregate2.

@Test
public void test_aggregate2() {
    IMap<Long, String> map = instance.getMap("source");
    // key is timestamp
    map.put(0L, "foo");
    map.put(2L, "baz");
    map.put(10L, "flush-item");
    IMap<Long, String> map2 = instance.getMap("source1");
    // key is timestamp
    map2.put(0L, "faa");
    map2.put(2L, "buu");
    map2.put(10L, "flush-item");
    Pipeline p = Pipeline.create();
    StreamStage<Entry<Long, String>> stage1 = p.drawFrom(Sources.<Long, String>mapJournal("source1", START_FROM_OLDEST));
    stage1.addTimestamps(Entry::getKey, 0);
    p.drawFrom(Sources.<Long, String>mapJournal("source", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).window(WindowDefinition.tumbling(2)).aggregate2(stage1, toTwoBags()).peek().drainTo(Sinks.list("sink"));
    instance.newJob(p);
    assertTrueEventually(() -> assertEquals(listToString(asList(new TimestampedItem<>(2, TwoBags.twoBags(asList(entry(0L, "foo")), asList(entry(0L, "faa")))), new TimestampedItem<>(4, TwoBags.twoBags(asList(entry(2L, "baz")), asList(entry(2L, "buu")))))), listToString(instance.getHazelcastInstance().getList("sink"))), 5);
}

Example 8 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast-jet by hazelcast.

the class WindowAggregateTransform_IntegrationTest method test_aggregate3.

@Test
public void test_aggregate3() {
    IMap<Long, String> map = instance.getMap("source");
    // key is timestamp
    map.put(0L, "foo");
    map.put(2L, "caz");
    map.put(10L, "flush-item");
    IMap<Long, String> map2 = instance.getMap("source1");
    // key is timestamp
    map2.put(0L, "faa");
    map2.put(2L, "cuu");
    map2.put(10L, "flush-item");
    IMap<Long, String> map3 = instance.getMap("source2");
    // key is timestamp
    map3.put(0L, "fzz");
    map3.put(2L, "ccc");
    map3.put(10L, "flush-item");
    Pipeline p = Pipeline.create();
    StreamStage<Entry<Long, String>> stage1 = p.drawFrom(Sources.<Long, String>mapJournal("source1", START_FROM_OLDEST));
    stage1.addTimestamps(Entry::getKey, 0);
    StreamStage<Entry<Long, String>> stage2 = p.drawFrom(Sources.<Long, String>mapJournal("source2", START_FROM_OLDEST));
    stage2.addTimestamps(Entry::getKey, 0);
    p.drawFrom(Sources.<Long, String>mapJournal("source", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).window(WindowDefinition.tumbling(2)).aggregate3(stage1, stage2, toThreeBags()).peek().drainTo(Sinks.list("sink"));
    instance.newJob(p);
    assertTrueEventually(() -> assertEquals(listToString(asList(new TimestampedItem<>(2, ThreeBags.threeBags(asList(entry(0L, "foo")), asList(entry(0L, "faa")), asList(entry(0L, "fzz")))), new TimestampedItem<>(4, ThreeBags.threeBags(asList(entry(2L, "caz")), asList(entry(2L, "cuu")), asList(entry(2L, "ccc")))))), listToString(instance.getHazelcastInstance().getList("sink"))), 5);
}

Example 9 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast-jet by hazelcast.

the class WindowAggregateTransform_IntegrationTest method test_aggregate2_with_aggregateBuilder.

@Test
public void test_aggregate2_with_aggregateBuilder() {
    IMap<Long, String> map = instance.getMap("source");
    // key is timestamp
    map.put(0L, "foo");
    map.put(2L, "baz");
    map.put(10L, "flush-item");
    IMap<Long, String> map2 = instance.getMap("source1");
    // key is timestamp
    map2.put(0L, "faa");
    map2.put(2L, "buu");
    map2.put(10L, "flush-item");
    Pipeline p = Pipeline.create();
    StreamStage<Entry<Long, String>> stage1 = p.drawFrom(Sources.<Long, String>mapJournal("source1", START_FROM_OLDEST));
    stage1.addTimestamps(Entry::getKey, 0);
    WindowAggregateBuilder<Entry<Long, String>> b = p.drawFrom(Sources.<Long, String>mapJournal("source", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).window(WindowDefinition.tumbling(2)).aggregateBuilder();
    Tag<Entry<Long, String>> tag0 = b.tag0();
    Tag<Entry<Long, String>> tag1 = b.add(stage1);
    b.build(AggregateOperation.withCreate(TwoBags::twoBags).andAccumulate(tag0, (acc, item0) -> acc.bag0().add(item0)).andAccumulate(tag1, (acc, item1) -> acc.bag1().add(item1)).andCombine(TwoBags::combineWith).andDeduct(TwoBags::deduct).andFinish(TwoBags::finish)).peek().drainTo(Sinks.list("sink"));
    instance.newJob(p);
    assertTrueEventually(() -> assertEquals(listToString(asList(new TimestampedItem<>(2, TwoBags.twoBags(asList(entry(0L, "foo")), asList(entry(0L, "faa")))), new TimestampedItem<>(4, TwoBags.twoBags(asList(entry(2L, "baz")), asList(entry(2L, "buu")))))), listToString(instance.getHazelcastInstance().getList("sink"))), 5);
}

Example 10 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast-jet by hazelcast.

the class WindowAggregateTransform_IntegrationTest method testWindowDefinition.

@Test
public void testWindowDefinition() {
    Pipeline p = Pipeline.create();
    SlidingWindowDef tumbling = WindowDefinition.tumbling(2);
    StageWithWindow<Entry<Long, String>> stage = p.drawFrom(Sources.<Long, String>mapJournal("source", START_FROM_OLDEST)).window(tumbling);
    assertEquals(tumbling, stage.windowDefinition());
}