Examples with Sources com.hazelcast.jet.pipeline.Sources used on opensource projects

Example 11 with Sources

use of com.hazelcast.jet.pipeline.Sources in project zeppelin by apache.

the class HazelcastJetInterpreterUtilsTest method testDisplayNetworkFromDAGUtil.

@Test
public void testDisplayNetworkFromDAGUtil() {
    Pipeline p = Pipeline.create();
    p.drawFrom(Sources.<String>list("text")).flatMap(word -> traverseArray(word.toLowerCase().split("\\W+"))).setName("flat traversing").filter(word -> !word.isEmpty()).groupingKey(wholeItem()).aggregate(counting()).drainTo(Sinks.map("counts"));
    assertEquals(NETWORK_RESULT_1, HazelcastJetInterpreterUtils.displayNetworkFromDAG(p.toDag()));
}

Example 12 with Sources

use of com.hazelcast.jet.pipeline.Sources in project hazelcast by hazelcast.

the class JobSerializerTest method when_serializerIsRegistered_then_itIsAvailableForLocalMapSource.

@Test
public void when_serializerIsRegistered_then_itIsAvailableForLocalMapSource() {
    Map<Integer, Value> map = client().getMap(SOURCE_MAP_NAME);
    map.putAll(ImmutableMap.of(1, new Value(1), 2, new Value(2)));
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(Sources.<Integer, Value>map(SOURCE_MAP_NAME)).map(entry -> entry.getValue().value()).writeTo(AssertionSinks.assertAnyOrder(asList(1, 2)));
    client().getJet().newJob(pipeline, jobConfig()).join();
}

Example 13 with Sources

use of com.hazelcast.jet.pipeline.Sources in project hazelcast by hazelcast.

the class JobSerializerTest method when_serializerIsRegistered_then_itIsAvailableForLocalCacheSource.

@Test
public void when_serializerIsRegistered_then_itIsAvailableForLocalCacheSource() {
    Cache<Integer, Value> map = client().getCacheManager().getCache(SOURCE_CACHE_NAME);
    map.putAll(ImmutableMap.of(1, new Value(1), 2, new Value(2)));
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(Sources.<Integer, Value>cache(SOURCE_CACHE_NAME)).map(entry -> entry.getValue().value()).writeTo(AssertionSinks.assertAnyOrder(asList(1, 2)));
    client().getJet().newJob(pipeline, jobConfig()).join();
}

Example 14 with Sources

use of com.hazelcast.jet.pipeline.Sources in project hazelcast-jet by hazelcast.

the class WindowAggregateTransform_IntegrationTest method testSession.

@Test
public void testSession() {
    IMap<Long, String> map = instance.getMap("source");
    map.put(0L, "foo");
    map.put(3L, "bar");
    map.put(4L, "baz");
    map.put(10L, "flush-item");
    Pipeline p = Pipeline.create();
    p.drawFrom(Sources.<Long, String>mapJournal("source", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).window(WindowDefinition.session(2)).aggregate(toSet(), (winStart, winEnd, result) -> new WindowResult<>(winStart, winEnd, "", result)).drainTo(Sinks.list("sink"));
    instance.newJob(p);
    assertTrueEventually(() -> assertEquals(listToString(asList(new WindowResult<>(0, 2, "", set(entry(0L, "foo"))), new WindowResult<>(3, 6, "", set(entry(3L, "bar"), entry(4L, "baz"))))), listToString(instance.getHazelcastInstance().getList("sink"))), 5);
}

Example 15 with Sources

use of com.hazelcast.jet.pipeline.Sources in project hazelcast-jet by hazelcast.

the class WindowGroupTransform_IntegrationTest method testSliding_windowFirst_aggregate2.

@Test
public void testSliding_windowFirst_aggregate2() {
    IMap<Long, String> map = instance.getMap("source");
    // key is timestamp
    map.put(0L, "foo");
    map.put(2L, "taz");
    map.put(10L, "flush-item");
    IMap<Long, String> map2 = instance.getMap("source1");
    // key is timestamp
    map2.put(0L, "faa");
    map2.put(2L, "tuu");
    map2.put(10L, "flush-item");
    Pipeline p = Pipeline.create();
    StreamStageWithGrouping<Entry<Long, String>, Character> stage1 = p.drawFrom(Sources.<Long, String>mapJournal("source1", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).groupingKey(entry -> entry.getValue().charAt(0));
    p.drawFrom(Sources.<Long, String>mapJournal("source", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).window(WindowDefinition.tumbling(2)).groupingKey(entry -> entry.getValue().charAt(0)).aggregate2(stage1, toTwoBags()).peek().drainTo(Sinks.list("sink"));
    instance.newJob(p);
    assertTrueEventually(() -> {
        assertEquals(listToString(asList(new TimestampedEntry<>(2, 'f', TwoBags.twoBags(asList(entry(0L, "foo")), asList(entry(0L, "faa")))), new TimestampedEntry<>(4, 't', TwoBags.twoBags(asList(entry(2L, "taz")), asList(entry(2L, "tuu")))))), listToString(instance.getHazelcastInstance().getList("sink")));
    }, 5);
}