Examples with BatchStage com.hazelcast.jet.pipeline.BatchStage used on opensource projects

Example 6 with BatchStage

use of com.hazelcast.jet.pipeline.BatchStage in project hazelcast by hazelcast.

the class PythonServiceTest method batchStage_mapUsingPython_onAllMembers.

@Test
@Category(NightlyTest.class)
public void batchStage_mapUsingPython_onAllMembers() {
    // Given
    PythonServiceConfig cfg = new PythonServiceConfig().setHandlerFile(baseDir + "/echo.py").setHandlerFunction("handle");
    List<String> items = IntStream.range(0, ITEM_COUNT).mapToObj(Integer::toString).collect(toList());
    Pipeline p = Pipeline.create();
    BatchStage<String> stage = p.readFrom(TestSources.items(items));
    // When
    BatchStage<String> mapped = stage.apply(mapUsingPythonBatch(x -> x, cfg)).setLocalParallelism(2);
    // Then
    mapped.writeTo(AssertionSinks.assertAnyOrder("Python didn't map the items correctly", items.stream().map(i -> "echo-" + i).collect(toList())));
    instance().getJet().newJob(p).join();
}

Example 7 with BatchStage

use of com.hazelcast.jet.pipeline.BatchStage in project hazelcast by hazelcast.

the class PythonServiceTest method batchStage_mapUsingPython.

@Test
@Category({ QuickTest.class, ParallelJVMTest.class })
public void batchStage_mapUsingPython() {
    // Given
    PythonServiceConfig cfg = new PythonServiceConfig().setBaseDir(baseDir.toString()).setHandlerModule("echo").setHandlerFunction("handle");
    List<String> items = IntStream.range(0, ITEM_COUNT).mapToObj(Integer::toString).collect(toList());
    Pipeline p = Pipeline.create();
    BatchStage<String> stage = p.readFrom(TestSources.items(items));
    // When
    BatchStage<String> mapped = stage.apply(mapUsingPythonBatch(cfg)).setLocalParallelism(2);
    // Then
    mapped.writeTo(AssertionSinks.assertAnyOrder("Python didn't map the items correctly", items.stream().map(i -> "echo-" + i).collect(toList())));
    instance().getJet().newJob(p).join();
}

Example 8 with BatchStage

use of com.hazelcast.jet.pipeline.BatchStage in project hazelcast by hazelcast.

the class ObservableResultsTest method multipleObservables.

@Test
public void multipleObservables() {
    Pipeline pipeline = Pipeline.create();
    BatchStage<Long> stage = pipeline.readFrom(TestSources.items(0L, 1L, 2L, 3L, 4L));
    TestObserver otherTestObserver = new TestObserver();
    Observable<Long> otherObservable = hz().getJet().getObservable("otherObservable");
    otherObservable.addObserver(otherTestObserver);
    stage.filter(i -> i % 2 == 0).writeTo(Sinks.observable(observableName));
    stage.filter(i -> i % 2 != 0).writeTo(Sinks.observable("otherObservable"));
    // when
    Job job = hz().getJet().newJob(pipeline);
    job.join();
    // then
    assertSortedValues(testObserver, 0L, 2L, 4L);
    assertError(testObserver, null);
    assertCompletions(testObserver, 1);
    // also
    assertSortedValues(otherTestObserver, 1L, 3L);
    assertError(otherTestObserver, null);
    assertCompletions(otherTestObserver, 1);
}

Example 9 with BatchStage

use of com.hazelcast.jet.pipeline.BatchStage in project hazelcast by hazelcast.

the class GrpcServiceTest method when_bidirectionalStreaming_distributed.

@Test
public void when_bidirectionalStreaming_distributed() throws IOException {
    // Given
    server = createServer(new GreeterServiceImpl());
    final int port = server.getPort();
    List<String> items = IntStream.range(0, ITEM_COUNT).mapToObj(Integer::toString).collect(toList());
    Pipeline p = Pipeline.create();
    BatchStageWithKey<String, String> stage = p.readFrom(TestSources.items(items)).groupingKey(i -> i);
    // When
    BatchStage<String> mapped = stage.mapUsingServiceAsync(bidirectionalStreaming(port), (service, key, item) -> {
        HelloRequest req = HelloRequest.newBuilder().setName(item).build();
        return service.call(req).thenApply(HelloReply::getMessage);
    });
    // Then
    mapped.writeTo(AssertionSinks.assertCollected(e -> {
        assertEquals("unexpected number of items received", ITEM_COUNT, e.size());
    }));
    instance().getJet().newJob(p).join();
}

Example 10 with BatchStage

use of com.hazelcast.jet.pipeline.BatchStage in project hazelcast by hazelcast.

the class GrpcServiceTest method when_repeatedBidirectionalStreaming_distributed.

@Test
public void when_repeatedBidirectionalStreaming_distributed() throws IOException {
    // Given
    server = createServer(new GreeterServiceImpl());
    final int port = server.getPort();
    List<String> items = IntStream.range(0, ITEM_COUNT).mapToObj(Integer::toString).collect(toList());
    Pipeline p = Pipeline.create();
    BatchStageWithKey<String, String> stage = p.readFrom(TestSources.items(items)).groupingKey(i -> i);
    // When
    BatchStage<String> mapped = stage.mapUsingServiceAsyncBatched(repeatedBidirectionalStreaming(port), 128, (service, itemList) -> {
        HelloRequestList req = HelloRequestList.newBuilder().addAllName(itemList).build();
        return service.call(req).thenApply(HelloReplyList::getMessageList);
    });
    // Then
    mapped.writeTo(AssertionSinks.assertCollected(e -> {
        assertEquals("unexpected number of items received", ITEM_COUNT, e.size());
    }));
    instance().getJet().newJob(p).join();
}