Examples with Sources com.hazelcast.jet.pipeline.Sources used on opensource projects

Example 16 with Sources

use of com.hazelcast.jet.pipeline.Sources in project hazelcast-jet by hazelcast.

the class WindowGroupTransform_IntegrationTest method testSliding_groupingFirst.

@Test
public void testSliding_groupingFirst() {
    Pipeline p = Pipeline.create();
    p.drawFrom(Sources.<Long, String>mapJournal("source", JournalInitialPosition.START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).groupingKey(entry -> entry.getValue().charAt(0)).window(WindowDefinition.tumbling(2)).aggregate(toSet()).drainTo(Sinks.list("sink"));
    testSliding(p);
}

Example 17 with Sources

use of com.hazelcast.jet.pipeline.Sources in project hazelcast-jet by hazelcast.

the class WindowGroupTransform_IntegrationTest method testSliding_windowFirst_aggregate3_with_aggregateBuilder.

@Test
public void testSliding_windowFirst_aggregate3_with_aggregateBuilder() {
    IMap<Long, String> map = instance.getMap("source");
    // key is timestamp
    map.put(0L, "foo");
    map.put(2L, "taz");
    map.put(10L, "flush-item");
    IMap<Long, String> map1 = instance.getMap("source1");
    // key is timestamp
    map1.put(0L, "faa");
    map1.put(2L, "tuu");
    map1.put(10L, "flush-item");
    IMap<Long, String> map2 = instance.getMap("source2");
    // key is timestamp
    map2.put(0L, "fzz");
    map2.put(2L, "tcc");
    map2.put(10L, "flush-item");
    Pipeline p = Pipeline.create();
    StreamStageWithGrouping<Entry<Long, String>, Character> stage1 = p.drawFrom(Sources.<Long, String>mapJournal("source1", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).groupingKey(entry -> entry.getValue().charAt(0));
    StreamStageWithGrouping<Entry<Long, String>, Character> stage2 = p.drawFrom(Sources.<Long, String>mapJournal("source2", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).groupingKey(entry -> entry.getValue().charAt(0));
    WindowGroupAggregateBuilder<Entry<Long, String>, Character> b = p.drawFrom(Sources.<Long, String>mapJournal("source", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).window(WindowDefinition.tumbling(2)).groupingKey(entry -> entry.getValue().charAt(0)).aggregateBuilder();
    Tag<Entry<Long, String>> tag0 = b.tag0();
    Tag<Entry<Long, String>> tag1 = b.add(stage1);
    Tag<Entry<Long, String>> tag2 = b.add(stage2);
    b.build(AggregateOperation.withCreate(ThreeBags::threeBags).andAccumulate(tag0, (acc, item0) -> acc.bag0().add(item0)).andAccumulate(tag1, (acc, item1) -> acc.bag1().add(item1)).andAccumulate(tag2, (acc, item2) -> acc.bag2().add(item2)).andCombine(ThreeBags::combineWith).andDeduct(ThreeBags::deduct).andFinish(ThreeBags::finish)).peek().drainTo(Sinks.list("sink"));
    instance.newJob(p);
    assertTrueEventually(() -> {
        assertEquals(listToString(asList(new TimestampedEntry<>(2, 'f', ThreeBags.threeBags(asList(entry(0L, "foo")), asList(entry(0L, "faa")), asList(entry(0L, "fzz")))), new TimestampedEntry<>(4, 't', ThreeBags.threeBags(asList(entry(2L, "taz")), asList(entry(2L, "tuu")), asList(entry(2L, "tcc")))))), listToString(instance.getHazelcastInstance().getList("sink")));
    }, 5);
}

Example 18 with Sources

use of com.hazelcast.jet.pipeline.Sources in project hazelcast-jet by hazelcast.

the class WriteFilePTest method when_toStringF_then_used.

@Test
public void when_toStringF_then_used() throws Exception {
    // Given
    Pipeline p = Pipeline.create();
    p.drawFrom(Sources.<String>list(list.getName())).drainTo(Sinks.files(directory.toString(), val -> Integer.toString(Integer.parseInt(val) - 1)));
    addItemsToList(1, 11);
    // When
    instance.newJob(p).join();
    // Then
    checkFileContents(StandardCharsets.UTF_8, 10);
}

Example 19 with Sources

use of com.hazelcast.jet.pipeline.Sources in project hazelcast-jet-reference-manual by hazelcast.

the class HelloWorld method main.

public static void main(String[] args) {
    // Create the specification of the computation pipeline. Note
    // it's a pure POJO: no instance of Jet needed to create it.
    Pipeline p = Pipeline.create();
    p.drawFrom(Sources.<String>list("text")).flatMap(word -> traverseArray(word.toLowerCase().split("\\W+"))).filter(word -> !word.isEmpty()).groupingKey(wholeItem()).aggregate(counting()).drainTo(Sinks.map("counts"));
    // Start Jet, populate the input list
    JetInstance jet = Jet.newJetInstance();
    try {
        List<String> text = jet.getList("text");
        text.add("hello world hello hello world");
        text.add("world world hello world");
        // Perform the computation
        jet.newJob(p).join();
        // Check the results
        Map<String, Long> counts = jet.getMap("counts");
        System.out.println("Count of hello: " + counts.get("hello"));
        System.out.println("Count of world: " + counts.get("world"));
    } finally {
        Jet.shutdownAll();
    }
}

Example 20 with Sources

use of com.hazelcast.jet.pipeline.Sources in project hazelcast-jet-reference-manual by hazelcast.

the class ImdgConnectors method s8.

static void s8() {
    ClientConfig clientConfig = new ClientConfig();
    // tag::s8[]
    Pipeline p = Pipeline.create();
    p.drawFrom(Sources.<Integer, String, Person>remoteMap("inputMap", clientConfig, e -> e.getValue().getAge() > 21, e -> e.getValue().getAge()));
// end::s8[]
}