use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.
the class HazelcastConnectorTest method when_readMap_and_writeMap.
@Test
public void when_readMap_and_writeMap() {
IMapJet<Integer, Integer> sourceMap = jetInstance.getMap(sourceName);
range(0, ENTRY_COUNT).forEach(i -> sourceMap.put(i, i));
DAG dag = new DAG();
Vertex source = dag.newVertex("source", readMapP(sourceName));
Vertex sink = dag.newVertex("sink", writeMapP(sinkName));
dag.edge(between(source, sink));
jetInstance.newJob(dag).join();
assertEquals(ENTRY_COUNT, jetInstance.getMap(sinkName).size());
}
use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.
the class HazelcastConnectorTest method when_readMap_withProjectionToNull_then_nullsSkipped.
@Test
public void when_readMap_withProjectionToNull_then_nullsSkipped() {
IMapJet<Integer, Entry<Integer, String>> sourceMap = jetInstance.getMap(sourceName);
range(0, ENTRY_COUNT).forEach(i -> sourceMap.put(i, entry(i, i % 2 == 0 ? null : String.valueOf(i))));
DAG dag = new DAG();
Vertex source = dag.newVertex("source", readMapP(sourceName, new TruePredicate<>(), Projections.singleAttribute("value")));
Vertex sink = dag.newVertex("sink", writeListP(sinkName));
dag.edge(between(source, sink));
jetInstance.newJob(dag).join();
checkContents_projectedToNull(sinkName);
}
use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.
the class HazelcastConnectorTest method when_streamMap_withProjectionToNull_then_nullsSkipped.
@Test
public void when_streamMap_withProjectionToNull_then_nullsSkipped() {
DAG dag = new DAG();
Vertex source = dag.newVertex("source", SourceProcessors.streamMapP(streamSourceName, mapPutEvents(), (EventJournalMapEvent<Integer, Entry<Integer, String>> entry) -> entry.getNewValue().getValue(), START_FROM_OLDEST, noWatermarks()));
Vertex sink = dag.newVertex("sink", writeListP(streamSinkName));
dag.edge(between(source, sink));
Job job = jetInstance.newJob(dag);
IMapJet<Integer, Entry<Integer, String>> sourceMap = jetInstance.getMap(streamSourceName);
range(0, ENTRY_COUNT).forEach(i -> sourceMap.put(i, entry(i, i % 2 == 0 ? null : String.valueOf(i))));
assertTrueEventually(() -> checkContents_projectedToNull(streamSinkName), 10);
job.cancel();
}
use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.
the class HazelcastConnectorTest method when_readCache_and_writeCache.
@Test
public void when_readCache_and_writeCache() {
ICache<Integer, Integer> sourceCache = jetInstance.getCacheManager().getCache(sourceName);
range(0, ENTRY_COUNT).forEach(i -> sourceCache.put(i, i));
DAG dag = new DAG();
Vertex source = dag.newVertex("source", readCacheP(sourceName));
Vertex sink = dag.newVertex("sink", writeCacheP(sinkName));
dag.edge(between(source, sink));
jetInstance.newJob(dag).join();
assertEquals(ENTRY_COUNT, jetInstance.getCacheManager().getCache(sinkName).size());
}
use of com.hazelcast.jet.core.DAG in project hazelcast-jet by hazelcast.
the class WordCountTest method testJet.
@Test
public void testJet() {
DAG dag = new DAG();
Vertex source = dag.newVertex("source", SourceProcessors.readMapP("words"));
Vertex tokenize = dag.newVertex("tokenize", flatMapP((Map.Entry<?, String> line) -> {
StringTokenizer s = new StringTokenizer(line.getValue());
return () -> s.hasMoreTokens() ? s.nextToken() : null;
}));
// word -> (word, count)
Vertex aggregateStage1 = dag.newVertex("aggregateStage1", aggregateByKeyP(singletonList(wholeItem()), counting(), Util::entry));
// (word, count) -> (word, count)
DistributedFunction<Entry, ?> getEntryKeyFn = Entry::getKey;
Vertex aggregateStage2 = dag.newVertex("aggregateStage2", aggregateByKeyP(singletonList(getEntryKeyFn), summingLong(Entry<String, Long>::getValue), Util::entry));
Vertex sink = dag.newVertex("sink", SinkProcessors.writeMapP("counts"));
dag.edge(between(source.localParallelism(1), tokenize)).edge(between(tokenize, aggregateStage1).partitioned(wholeItem(), HASH_CODE)).edge(between(aggregateStage1, aggregateStage2).distributed().partitioned(entryKey())).edge(between(aggregateStage2, sink.localParallelism(1)));
benchmark("jet", () -> instance.newJob(dag).join());
assertCounts(instance.getMap("counts"));
}
Aggregations