Examples with FunctionEx com.hazelcast.function.FunctionEx used on opensource projects

Example 16 with FunctionEx

use of com.hazelcast.function.FunctionEx in project hazelcast by hazelcast.

the class SourceBuilderTest method stream_socketSource_withTimestamps_andLateness.

@Test
public void stream_socketSource_withTimestamps_andLateness() throws IOException {
    // Given
    try (ServerSocket serverSocket = new ServerSocket(0)) {
        startServer(serverSocket);
        // When
        int localPort = serverSocket.getLocalPort();
        FunctionEx<String, Long> timestampFn = line -> Long.valueOf(line.substring(LINE_PREFIX.length()));
        int lateness = 10;
        long lastExpectedTs = itemCount - lateness;
        StreamSource<String> socketSource = SourceBuilder.timestampedStream("socket-source-with-timestamps", ctx -> socketReader(localPort)).<String>fillBufferFn((in, buf) -> {
            String line = in.readLine();
            if (line != null) {
                long ts = timestampFn.apply(line);
                buf.add(line, ts);
                if (ts >= lastExpectedTs) {
                    System.out.println(line);
                }
            }
        }).destroyFn(BufferedReader::close).build();
        // Then
        Pipeline p = Pipeline.create();
        p.readFrom(socketSource).withNativeTimestamps(lateness).window(tumbling(1)).aggregate(AggregateOperations.counting()).writeTo(sinkList());
        hz().getJet().newJob(p);
        List<WindowResult<Long>> expected = LongStream.range(1, itemCount - lateness).mapToObj(i -> new WindowResult<>(i - 1, i, 1L)).collect(toList());
        assertTrueEventually(() -> assertEquals(expected, new ArrayList<>(sinkList)), 10);
    }
}

Example 17 with FunctionEx

use of com.hazelcast.function.FunctionEx in project hazelcast by hazelcast.

the class ManagedContextTest method testServices.

private void testServices(FunctionEx<ProcessorSupplier.Context, ? extends AnotherTestServiceContext> serviceSupplier) {
    // Given
    ServiceFactory<?, ? extends AnotherTestServiceContext> serviceFactory = ServiceFactories.sharedService(serviceSupplier);
    Pipeline p = Pipeline.create();
    p.readFrom(TestSources.items("item")).mapUsingService(serviceFactory, (c, item) -> item + c.injectedValue).writeTo(assertAnyOrder(singletonList("item" + INJECTED_VALUE)));
    // When
    hz.getJet().newJob(p).join();
}

Example 18 with FunctionEx

use of com.hazelcast.function.FunctionEx in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_peekAndRebalanceAndMap_then_dagEdgeDistributed.

@Test
public void when_peekAndRebalanceAndMap_then_dagEdgeDistributed() {
    // Given
    List<Integer> input = sequence(itemCount);
    BatchStage<Integer> srcStage = batchStageFromList(input);
    FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
    // When
    BatchStage<String> mapped = srcStage.peek().rebalance().map(formatFn);
    // Then
    mapped.writeTo(sink);
    DAG dag = p.toDag();
    Edge srcToMap = dag.getInboundEdges("map").get(0);
    assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", srcToMap.getPartitioner());
    execute();
    assertEquals(streamToString(input.stream(), formatFn), streamToString(sinkStreamOf(String.class), identity()));
}

Example 19 with FunctionEx

use of com.hazelcast.function.FunctionEx in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_rebalanceAndGroupAggregateBuilderWithComplexAggrOp_then_singleStageAggregation.

@Test
public void when_rebalanceAndGroupAggregateBuilderWithComplexAggrOp_then_singleStageAggregation() {
    // Given
    FunctionEx<Integer, Integer> keyFn = i -> i % 10;
    Iterator<Integer> sequence = IntStream.iterate(0, i -> i + 1).boxed().iterator();
    List<Integer> input0 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    List<Integer> input1 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    List<Integer> input2 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    BatchStageWithKey<Integer, Integer> stage0 = batchStageFromList(input0).groupingKey(keyFn);
    BatchStageWithKey<Integer, Integer> stage1 = batchStageFromList(input1).groupingKey(keyFn);
    BatchStageWithKey<Integer, Integer> stage2Rebalanced = batchStageFromList(input2).rebalance().groupingKey(keyFn);
    // When
    GroupAggregateBuilder1<Integer, Integer> b = stage0.aggregateBuilder();
    Tag<Integer> tag0_in = b.tag0();
    Tag<Integer> tag1_in = b.add(stage1);
    Tag<Integer> tag2_in = b.add(stage2Rebalanced);
    CoAggregateOperationBuilder agb = coAggregateOperationBuilder();
    Tag<Long> tag0 = agb.add(tag0_in, SUMMING);
    Tag<Long> tag1 = agb.add(tag1_in, SUMMING);
    Tag<Long> tag2 = agb.add(tag2_in, SUMMING);
    AggregateOperation<Object[], ItemsByTag> aggrOp = agb.build();
    BatchStage<Entry<Integer, ItemsByTag>> aggregated = b.build(aggrOp);
    // Then
    aggregated.writeTo(sink);
    assertSingleStageAggregation();
    execute();
    Collector<Integer, ?, Map<Integer, Long>> groupAndSum = groupingBy(keyFn, summingLong(i -> i));
    Map<Integer, Long> expectedMap0 = input0.stream().collect(groupAndSum);
    Map<Integer, Long> expectedMap1 = input1.stream().collect(groupAndSum);
    Map<Integer, Long> expectedMap2 = input2.stream().collect(groupAndSum);
    assertEquals(streamToString(expectedMap0.entrySet().stream(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue(), expectedMap1.get(e.getKey()), expectedMap2.get(e.getKey())))), streamToString(this.<Integer, ItemsByTag>sinkStreamOfEntry(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue().get(tag0), e.getValue().get(tag1), e.getValue().get(tag2)))));
}

Example 20 with FunctionEx

use of com.hazelcast.function.FunctionEx in project hazelcast by hazelcast.

the class RebalanceStreamStageTest method when_peekAndRebalanceByKeyAndMap_then_dagEdgePartitionedDistributed.

@Test
public void when_peekAndRebalanceByKeyAndMap_then_dagEdgePartitionedDistributed() {
    // Given
    List<Integer> input = sequence(itemCount);
    StreamStage<Integer> srcStage = streamStageFromList(input);
    FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
    // When
    StreamStage<String> mapped = srcStage.peek().rebalance(i -> i).map(formatFn);
    // Then
    mapped.writeTo(sink);
    DAG dag = p.toDag();
    Edge srcToMap = dag.getInboundEdges("map").get(0);
    assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
    assertNotNull("Rebalanced by key, the edge must be partitioned", srcToMap.getPartitioner());
}