Examples with IListJet com.hazelcast.jet.IListJet used on opensource projects

Example 1 with IListJet

use of com.hazelcast.jet.IListJet in project hazelcast-jet by hazelcast.

the class ReadHdfsPTest method testJus.

@Test
public void testJus() {
    IListJet sink = DistributedStream.fromSource(instance, HdfsSources.hdfs(jobConf, mapperType.mapper), false).collect(DistributedCollectors.toIList("sink"));
    assertEquals(expectedSinkSize(), sink.size());
}

Example 2 with IListJet

use of com.hazelcast.jet.IListJet in project hazelcast-jet by hazelcast.

the class SourcesTest method testFile.

@Test
public void testFile() throws IOException {
    File dir = createTempFile();
    IListJet<String> sink = DistributedStream.fromSource(getInstance(), Sources.files(dir.getAbsolutePath()), false).flatMap(line -> Arrays.stream(line.split(" "))).collect(DistributedCollectors.toIList(dir.getName()));
    assertEquals(10, sink.size());
    deleteTempFile(dir);
}

Example 3 with IListJet

use of com.hazelcast.jet.IListJet in project hazelcast-jet by hazelcast.

the class AggregateTransform_IntegrationTest method test_aggregate3_with_aggBuilder.

@Test
public void test_aggregate3_with_aggBuilder() {
    // Given
    JetInstance instance = createJetMember();
    IListJet<Integer> list = instance.getList("list");
    list.add(1);
    list.add(2);
    list.add(3);
    IListJet<String> list1 = instance.getList("list1");
    list1.add("a");
    list1.add("b");
    list1.add("c");
    IListJet<Double> list2 = instance.getList("list2");
    list2.add(6.0d);
    list2.add(7.0d);
    list2.add(8.0d);
    // When
    Pipeline p = Pipeline.create();
    BatchStage<Integer> stage0 = p.drawFrom(Sources.list("list"));
    BatchStage<String> stage1 = p.drawFrom(Sources.list("list1"));
    BatchStage<Double> stage2 = p.drawFrom(Sources.list("list2"));
    AggregateBuilder<Integer> builder = stage0.aggregateBuilder();
    Tag<Integer> tag0 = builder.tag0();
    Tag<String> tag1 = builder.add(stage1);
    Tag<Double> tag2 = builder.add(stage2);
    BatchStage<ThreeBags> resultStage = builder.build(AggregateOperation.withCreate(ThreeBags::threeBags).andAccumulate(tag0, (acc, item0) -> acc.bag0().add(item0)).andAccumulate(tag1, (acc, item1) -> acc.bag1().add(item1)).andAccumulate(tag2, (acc, item2) -> acc.bag2().add(item2)).andCombine(ThreeBags::combineWith).andDeduct(ThreeBags::deduct).andFinish(ThreeBags::finish));
    resultStage.drainTo(Sinks.list("sink"));
    instance.newJob(p).join();
    // Then
    ThreeBags threeBags = (ThreeBags) instance.getHazelcastInstance().getList("sink").iterator().next();
    assertNotNull(threeBags);
    sort(threeBags);
    assertEquals(threeBags(list, list1, list2), threeBags);
}

Example 4 with IListJet

use of com.hazelcast.jet.IListJet in project hazelcast-jet by hazelcast.

the class HazelcastConnectorTest method when_readMap_withPredicateAndDistributedFunction.

@Test
public void when_readMap_withPredicateAndDistributedFunction() {
    IMapJet<Integer, Integer> sourceMap = jetInstance.getMap(sourceName);
    range(0, ENTRY_COUNT).forEach(i -> sourceMap.put(i, i));
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", readMapP(sourceName, e -> !e.getKey().equals(0), Map.Entry::getKey));
    Vertex sink = dag.newVertex("sink", writeListP(sinkName));
    dag.edge(between(source, sink));
    jetInstance.newJob(dag).join();
    IListJet<Object> list = jetInstance.getList(sinkName);
    assertEquals(ENTRY_COUNT - 1, list.size());
    assertFalse(list.contains(0));
    assertTrue(list.contains(1));
}