Examples with Pipeline com.hazelcast.jet.pipeline.Pipeline used on opensource projects

Example 56 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class ReadHadoopPTest method testReadHdfs.

@Test
public void testReadHdfs() {
    IList<Object> sinkList = instance().getList(randomName());
    Pipeline p = Pipeline.create();
    p.readFrom(HadoopSources.inputFormat(jobConf, projectionType.mapper)).setLocalParallelism(4).writeTo(Sinks.list(sinkList)).setLocalParallelism(1);
    instance().getJet().newJob(p).join();
    int expected = paths.size() * ENTRIES.length * (sharedFileSystem ? 1 : 2);
    assertEquals(projectionType == CUSTOM_WITH_NULLS ? expected / 2 : expected, sinkList.size());
    assertTrue(sinkList.get(0).toString().contains("value"));
}

Example 57 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class Hz3EnrichmentTest method testMapUsingReplicatedMap.

@Test
public void testMapUsingReplicatedMap() {
    ReplicatedMap<Object, Object> map = hz3.getReplicatedMap("test-replicated-map");
    map.put(1, "a");
    map.put(2, "b");
    HazelcastInstance hz = createHazelcastInstance();
    IList<String> results = hz.getList("result-list");
    Pipeline p = Pipeline.create();
    ServiceFactory<Hz3MapAdapter, Map<Integer, String>> hz3MapSF = hz3ReplicatedMapServiceFactory("test-replicated-map", HZ3_CLIENT_CONFIG);
    BiFunctionEx<? super Map<Integer, String>, ? super Integer, String> mapFn = mapUsingIMap(FunctionEx.identity(), (Integer i, String s) -> s);
    BatchStage<String> mapStage = p.readFrom(TestSources.items(1, 2, 3)).mapUsingService(hz3MapSF, mapFn);
    mapStage.writeTo(Sinks.list(results));
    JobConfig config = getJobConfig(mapStage.name());
    hz.getJet().newJob(p, config).join();
    assertThat(results).containsOnly("a", "b");
}

Example 58 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class Hz3SinksTest method testMapSink.

@Test
public void testMapSink() {
    HazelcastInstance hz = createHazelcastInstance();
    Pipeline p = Pipeline.create();
    BatchSource<SimpleEntry<Integer, String>> source = TestSources.items(new SimpleEntry<>(1, "a"), new SimpleEntry<>(2, "b"));
    Sink<Map.Entry<Integer, String>> sink = Hz3Sinks.remoteMap("test-map", HZ3_CLIENT_CONFIG);
    p.readFrom(source).writeTo(sink);
    JobConfig config = getJobConfig(sink.name());
    hz.getJet().newJob(p, config).join();
    Map<Integer, String> testMap = hz3.getMap("test-map");
    assertThat(testMap).containsOnly(entry(1, "a"), entry(2, "b"));
}

Example 59 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class GrpcServiceTest method when_unary_distributed.

@Test
public void when_unary_distributed() throws IOException {
    // Given
    server = createServer(new GreeterServiceImpl());
    final int port = server.getPort();
    List<String> items = IntStream.range(0, ITEM_COUNT).mapToObj(Integer::toString).collect(toList());
    Pipeline p = Pipeline.create();
    BatchStageWithKey<String, String> stage = p.readFrom(TestSources.items(items)).groupingKey(i -> i);
    // When
    BatchStage<String> mapped = stage.mapUsingServiceAsync(unary(port), (service, key, item) -> {
        HelloRequest req = HelloRequest.newBuilder().setName(item).build();
        return service.call(req).thenApply(HelloReply::getMessage);
    });
    // Then
    mapped.writeTo(AssertionSinks.assertCollected(e -> {
        assertEquals("unexpected number of items received", ITEM_COUNT, e.size());
    }));
    instance().getJet().newJob(p).join();
}

Example 60 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class GrpcServiceTest method when_unary_with_faultyService.

@Test
public void when_unary_with_faultyService() throws IOException {
    // Given
    server = createServer(new FaultyGreeterServiceImpl());
    final int port = server.getPort();
    Pipeline p = Pipeline.create();
    BatchStage<String> source = p.readFrom(TestSources.items("one", "two", "three", "four"));
    // When
    BatchStage<String> mapped = source.mapUsingServiceAsync(unary(port), (service, input) -> {
        HelloRequest request = HelloRequest.newBuilder().setName(input).build();
        return service.call(request).thenApply(HelloReply::getMessage);
    });
    // Then
    mapped.writeTo(Sinks.noop());
    try {
        instance().getJet().newJob(p).join();
        fail("Job should have failed");
    } catch (Exception e) {
        Throwable ex = ExceptionUtil.peel(e);
        assertThat(ex.getMessage()).contains("com.hazelcast.jet.grpc.impl.StatusRuntimeExceptionJet");
    }
}