Examples with BatchStage com.hazelcast.jet.pipeline.BatchStage used on opensource projects

Example 1 with BatchStage

use of com.hazelcast.jet.pipeline.BatchStage in project hazelcast-jet by hazelcast.

the class AggregateTransform_IntegrationTest method test_aggregate3_with_aggBuilder.

@Test
public void test_aggregate3_with_aggBuilder() {
    // Given
    JetInstance instance = createJetMember();
    IListJet<Integer> list = instance.getList("list");
    list.add(1);
    list.add(2);
    list.add(3);
    IListJet<String> list1 = instance.getList("list1");
    list1.add("a");
    list1.add("b");
    list1.add("c");
    IListJet<Double> list2 = instance.getList("list2");
    list2.add(6.0d);
    list2.add(7.0d);
    list2.add(8.0d);
    // When
    Pipeline p = Pipeline.create();
    BatchStage<Integer> stage0 = p.drawFrom(Sources.list("list"));
    BatchStage<String> stage1 = p.drawFrom(Sources.list("list1"));
    BatchStage<Double> stage2 = p.drawFrom(Sources.list("list2"));
    AggregateBuilder<Integer> builder = stage0.aggregateBuilder();
    Tag<Integer> tag0 = builder.tag0();
    Tag<String> tag1 = builder.add(stage1);
    Tag<Double> tag2 = builder.add(stage2);
    BatchStage<ThreeBags> resultStage = builder.build(AggregateOperation.withCreate(ThreeBags::threeBags).andAccumulate(tag0, (acc, item0) -> acc.bag0().add(item0)).andAccumulate(tag1, (acc, item1) -> acc.bag1().add(item1)).andAccumulate(tag2, (acc, item2) -> acc.bag2().add(item2)).andCombine(ThreeBags::combineWith).andDeduct(ThreeBags::deduct).andFinish(ThreeBags::finish));
    resultStage.drainTo(Sinks.list("sink"));
    instance.newJob(p).join();
    // Then
    ThreeBags threeBags = (ThreeBags) instance.getHazelcastInstance().getList("sink").iterator().next();
    assertNotNull(threeBags);
    sort(threeBags);
    assertEquals(threeBags(list, list1, list2), threeBags);
}

Example 2 with BatchStage

use of com.hazelcast.jet.pipeline.BatchStage in project hazelcast-jet-reference-manual by hazelcast.

the class ImplementAggregation method s2.

static void s2() {
    // tag::s2[]
    Pipeline p = Pipeline.create();
    BatchStage<PageVisit> pageVisit = p.drawFrom(Sources.list("pageVisit"));
    BatchStage<AddToCart> addToCart = p.drawFrom(Sources.list("addToCart"));
    BatchStage<Payment> payment = p.drawFrom(Sources.list("payment"));
    AggregateOperation3<PageVisit, AddToCart, Payment, LongAccumulator[], long[]> aggrOp = AggregateOperation.withCreate(() -> new LongAccumulator[] { new LongAccumulator(), new LongAccumulator(), new LongAccumulator() }).<PageVisit>andAccumulate0((accs, pv) -> accs[0].add(pv.loadTime())).<AddToCart>andAccumulate1((accs, atc) -> accs[1].add(atc.quantity())).<Payment>andAccumulate2((accs, pm) -> accs[2].add(pm.amount())).andCombine((accs1, accs2) -> {
        accs1[0].add(accs2[0]);
        accs1[1].add(accs2[1]);
        accs1[2].add(accs2[2]);
    }).andFinish(accs -> new long[] { accs[0].get(), accs[1].get(), accs[2].get() });
    BatchStage<Entry<Integer, long[]>> coGrouped = pageVisit.groupingKey(PageVisit::userId).aggregate3(addToCart.groupingKey(AddToCart::userId), payment.groupingKey(Payment::userId), aggrOp);
// end::s2[]
}

Example 3 with BatchStage

use of com.hazelcast.jet.pipeline.BatchStage in project hazelcast by hazelcast.

the class GrpcServiceTest method when_unary_distributed.

@Test
public void when_unary_distributed() throws IOException {
    // Given
    server = createServer(new GreeterServiceImpl());
    final int port = server.getPort();
    List<String> items = IntStream.range(0, ITEM_COUNT).mapToObj(Integer::toString).collect(toList());
    Pipeline p = Pipeline.create();
    BatchStageWithKey<String, String> stage = p.readFrom(TestSources.items(items)).groupingKey(i -> i);
    // When
    BatchStage<String> mapped = stage.mapUsingServiceAsync(unary(port), (service, key, item) -> {
        HelloRequest req = HelloRequest.newBuilder().setName(item).build();
        return service.call(req).thenApply(HelloReply::getMessage);
    });
    // Then
    mapped.writeTo(AssertionSinks.assertCollected(e -> {
        assertEquals("unexpected number of items received", ITEM_COUNT, e.size());
    }));
    instance().getJet().newJob(p).join();
}

Example 4 with BatchStage

use of com.hazelcast.jet.pipeline.BatchStage in project hazelcast by hazelcast.

the class GrpcServiceTest method whenNotAsync_bidirectionalStreaming_distributed.

@Test
public void whenNotAsync_bidirectionalStreaming_distributed() throws IOException {
    // Given
    server = createServer(new GreeterServiceImpl());
    final int port = server.getPort();
    List<String> items = IntStream.range(0, ITEM_COUNT).mapToObj(Integer::toString).collect(toList());
    Pipeline p = Pipeline.create();
    BatchStageWithKey<String, String> stage = p.readFrom(TestSources.items(items)).groupingKey(i -> i);
    // When
    BatchStage<String> mapped = stage.mapUsingService(bidirectionalStreaming(port), (service, key, item) -> {
        HelloRequest req = HelloRequest.newBuilder().setName(item).build();
        return service.call(req).thenApply(HelloReply::getMessage).get();
    });
    // Then
    mapped.writeTo(AssertionSinks.assertCollected(e -> {
        assertEquals("unexpected number of items received", ITEM_COUNT, e.size());
    }));
    instance().getJet().newJob(p).join();
}

Example 5 with BatchStage

use of com.hazelcast.jet.pipeline.BatchStage in project hazelcast by hazelcast.

the class GrpcServiceTest method when_repeatedBidirectionalStreaming.

@Test
public void when_repeatedBidirectionalStreaming() throws IOException {
    // Given
    server = createServer(new GreeterServiceImpl());
    final int port = server.getPort();
    List<String> items = IntStream.range(0, ITEM_COUNT).mapToObj(Integer::toString).collect(toList());
    Pipeline p = Pipeline.create();
    BatchStage<String> stage = p.readFrom(TestSources.items(items));
    // When
    BatchStage<String> mapped = stage.mapUsingServiceAsyncBatched(repeatedBidirectionalStreaming(port), 128, (service, itemList) -> {
        HelloRequestList req = HelloRequestList.newBuilder().addAllName(itemList).build();
        return service.call(req).thenApply(HelloReplyList::getMessageList);
    });
    // Then
    List<String> expected = IntStream.range(0, ITEM_COUNT).boxed().map(t -> "Hello " + Integer.toString(t)).collect(toList());
    mapped.writeTo(AssertionSinks.assertAnyOrder(expected));
    instance().getJet().newJob(p).join();
}