Examples with Util com.hazelcast.jet.Util used on opensource projects

Example 1 with Util

use of com.hazelcast.jet.Util in project hazelcast-jet by hazelcast.

the class ReadFilesPTest method buildDag.

private DAG buildDag(String glob) {
    if (glob == null) {
        glob = "*";
    }
    DAG dag = new DAG();
    Vertex reader = dag.newVertex("reader", readFilesP(directory.getPath(), StandardCharsets.UTF_8, glob, Util::entry)).localParallelism(1);
    Vertex writer = dag.newVertex("writer", writeListP(list.getName())).localParallelism(1);
    dag.edge(between(reader, writer));
    return dag;
}

Example 2 with Util

use of com.hazelcast.jet.Util in project hazelcast by hazelcast.

the class StreamFilesP_integrationTest method buildDag.

private DAG buildDag() {
    DAG dag = new DAG();
    Vertex reader = dag.newVertex("reader", streamFilesP(directory.getPath(), UTF_8, "*", false, Util::entry)).localParallelism(1);
    Vertex writer = dag.newVertex("writer", writeListP(list.getName())).localParallelism(1);
    dag.edge(between(reader, writer));
    return dag;
}

Example 3 with Util

use of com.hazelcast.jet.Util in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_hashJoinRebalanceEnrichingStage_then_noEffect.

@Test
public void when_hashJoinRebalanceEnrichingStage_then_noEffect() {
    // Given
    List<Integer> input = sequence(itemCount);
    String prefix = "value-";
    BatchStage<Integer> mainStage = batchStageFromList(input);
    BatchStage<Entry<Integer, String>> enrichingStageRebalanced = batchStageFromList(input).map(i -> entry(i, prefix + i)).rebalance();
    // When
    BatchStage<Entry<Integer, String>> joined = mainStage.hashJoin(enrichingStageRebalanced, joinMapEntries(wholeItem()), Util::entry);
    // Then
    joined.writeTo(sink);
    DAG dag = p.toDag();
    Edge mapToJoin = dag.getInboundEdges("2-way hash-join-collector1").get(0);
    assertTrue("Edge into a hash-join collector vertex must be distributed", mapToJoin.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", mapToJoin.getPartitioner());
    Edge stage0ToJoin = dag.getInboundEdges("2-way hash-join-joiner").get(0);
    assertFalse("Didn't rebalance this stage, why is its edge distributed?", stage0ToJoin.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", stage0ToJoin.getPartitioner());
    execute();
    Function<Entry<Integer, String>, String> formatFn = e -> String.format("(%04d, %s)", e.getKey(), e.getValue());
    assertEquals(streamToString(input.stream().map(i -> tuple2(i, prefix + i)), formatFn), streamToString(sinkStreamOfEntry(), formatFn));
}

Example 4 with Util

use of com.hazelcast.jet.Util in project hazelcast by hazelcast.

the class StreamFilesPTest method when_metaSupplier_then_returnsCorrectProcessors.

@Test
public void when_metaSupplier_then_returnsCorrectProcessors() throws Exception {
    ProcessorMetaSupplier metaSupplier = streamFilesP(workDir.getAbsolutePath(), UTF_8, "*", false, Util::entry);
    Address a = new Address();
    ProcessorSupplier supplier = metaSupplier.get(singletonList(a)).apply(a);
    supplier.init(new TestProcessorContext());
    assertEquals(1, supplier.get(1).size());
    supplier.close(null);
}

Example 5 with Util

use of com.hazelcast.jet.Util in project hazelcast-jet by hazelcast.

the class StreamFilesPTest method when_metaSupplier_then_returnsCorrectProcessors.

@Test
public void when_metaSupplier_then_returnsCorrectProcessors() {
    ProcessorMetaSupplier metaSupplier = streamFilesP(workDir.getAbsolutePath(), UTF_8, "*", Util::entry);
    Address a = new Address();
    ProcessorSupplier supplier = metaSupplier.get(singletonList(a)).apply(a);
    supplier.init(new TestProcessorContext());
    assertEquals(1, supplier.get(1).size());
    supplier.close(null);
}