Examples with StreamStage com.hazelcast.jet.pipeline.StreamStage used on opensource projects

Example 1 with StreamStage

use of com.hazelcast.jet.pipeline.StreamStage in project hazelcast-jet by hazelcast.

the class WindowAggregateTransform_IntegrationTest method test_aggregate2_with_aggregateBuilder.

@Test
public void test_aggregate2_with_aggregateBuilder() {
    IMap<Long, String> map = instance.getMap("source");
    // key is timestamp
    map.put(0L, "foo");
    map.put(2L, "baz");
    map.put(10L, "flush-item");
    IMap<Long, String> map2 = instance.getMap("source1");
    // key is timestamp
    map2.put(0L, "faa");
    map2.put(2L, "buu");
    map2.put(10L, "flush-item");
    Pipeline p = Pipeline.create();
    StreamStage<Entry<Long, String>> stage1 = p.drawFrom(Sources.<Long, String>mapJournal("source1", START_FROM_OLDEST));
    stage1.addTimestamps(Entry::getKey, 0);
    WindowAggregateBuilder<Entry<Long, String>> b = p.drawFrom(Sources.<Long, String>mapJournal("source", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).window(WindowDefinition.tumbling(2)).aggregateBuilder();
    Tag<Entry<Long, String>> tag0 = b.tag0();
    Tag<Entry<Long, String>> tag1 = b.add(stage1);
    b.build(AggregateOperation.withCreate(TwoBags::twoBags).andAccumulate(tag0, (acc, item0) -> acc.bag0().add(item0)).andAccumulate(tag1, (acc, item1) -> acc.bag1().add(item1)).andCombine(TwoBags::combineWith).andDeduct(TwoBags::deduct).andFinish(TwoBags::finish)).peek().drainTo(Sinks.list("sink"));
    instance.newJob(p);
    assertTrueEventually(() -> assertEquals(listToString(asList(new TimestampedItem<>(2, TwoBags.twoBags(asList(entry(0L, "foo")), asList(entry(0L, "faa")))), new TimestampedItem<>(4, TwoBags.twoBags(asList(entry(2L, "baz")), asList(entry(2L, "buu")))))), listToString(instance.getHazelcastInstance().getList("sink"))), 5);
}

Example 2 with StreamStage

use of com.hazelcast.jet.pipeline.StreamStage in project hazelcast-jet by hazelcast.

the class GrAggBuilder method buildStream.

@SuppressWarnings("unchecked")
public <A, R, OUT> StreamStage<OUT> buildStream(@Nonnull AggregateOperation<A, ? extends R> aggrOp, @Nonnull KeyedWindowResultFunction<? super K, ? super R, OUT> mapToOutputFn) {
    List<Transform> upstreamTransforms = upstreamStages.stream().map(s -> s.transform).collect(toList());
    JetEventFunctionAdapter fnAdapter = ADAPT_TO_JET_EVENT;
    // Avoided Stream API here due to static typing issues
    List<DistributedFunction<?, ? extends K>> adaptedKeyFns = new ArrayList<>();
    for (DistributedFunction keyFn : keyFns) {
        adaptedKeyFns.add(adaptKeyFn(keyFn));
    }
    Transform transform = new WindowGroupTransform<K, A, R, JetEvent<OUT>>(upstreamTransforms, wDef, adaptedKeyFns, adaptAggregateOperation(aggrOp), fnAdapter.adaptKeyedWindowResultFn(mapToOutputFn));
    pipelineImpl.connect(upstreamTransforms, transform);
    return new StreamStageImpl<>(transform, fnAdapter, pipelineImpl);
}

Example 3 with StreamStage

use of com.hazelcast.jet.pipeline.StreamStage in project hazelcast by hazelcast.

the class PythonServiceTest method streamStage_mapUsingPython_withCleanupScript.

@Test
@Category(NightlyTest.class)
public void streamStage_mapUsingPython_withCleanupScript() throws Exception {
    // Given
    String baseDirStr = baseDir.toString();
    String outcomeFilename = "cleanup_outcome.txt";
    installFileToBaseDir(String.format("echo 'Cleanup script running'%n" + "echo 'cleanup.sh' executed > %s/%s%n", baseDirStr, outcomeFilename), "cleanup.sh");
    PythonServiceConfig cfg = new PythonServiceConfig().setBaseDir(baseDirStr).setHandlerModule("echo").setHandlerFunction("handle");
    List<String> items = singletonList("item-0");
    Pipeline p = Pipeline.create();
    StreamStage<String> stage = p.readFrom(TestSources.items(items)).addTimestamps(x -> 0, 0);
    // When
    StreamStage<String> mapped = stage.apply(mapUsingPython(cfg)).setLocalParallelism(2);
    // Then
    mapped.writeTo(AssertionSinks.assertAnyOrder("Python didn't map the items correctly", items.stream().map(i -> "echo-" + i).collect(toList())));
    instance().getJet().newJob(p).join();
    assertTrue("Cleanup script didn't run", new File(baseDir, outcomeFilename).isFile());
}

Example 4 with StreamStage

use of com.hazelcast.jet.pipeline.StreamStage in project hazelcast by hazelcast.

the class PythonServiceTest method streamStage_mapUsingPython_withInitScript.

@Test
@Category(NightlyTest.class)
public void streamStage_mapUsingPython_withInitScript() throws Exception {
    // Given
    String baseDirStr = baseDir.toString();
    String outcomeFilename = "init_outcome.txt";
    installFileToBaseDir(String.format("echo 'Init script running'%n" + "echo 'init.sh' executed > %s/%s%n", baseDirStr, outcomeFilename), "init.sh");
    PythonServiceConfig cfg = new PythonServiceConfig().setBaseDir(baseDirStr).setHandlerModule("echo").setHandlerFunction("handle");
    List<String> items = singletonList("item-0");
    Pipeline p = Pipeline.create();
    StreamStage<String> stage = p.readFrom(TestSources.items(items)).addTimestamps(x -> 0, 0);
    // When
    StreamStage<String> mapped = stage.apply(mapUsingPython(cfg)).setLocalParallelism(2);
    // Then
    mapped.writeTo(AssertionSinks.assertAnyOrder("Python didn't map the items correctly", items.stream().map(i -> "echo-" + i).collect(toList())));
    instance().getJet().newJob(p).join();
    assertTrue("Init script didn't run", new File(baseDir, outcomeFilename).isFile());
}

Example 5 with StreamStage

use of com.hazelcast.jet.pipeline.StreamStage in project hazelcast by hazelcast.

the class PythonServiceTest method streamStage_mapUsingPython_withBaseDir.

@Test
@Category(NightlyTest.class)
public void streamStage_mapUsingPython_withBaseDir() {
    // Given
    PythonServiceConfig cfg = new PythonServiceConfig().setBaseDir(baseDir.toString()).setHandlerModule("echo").setHandlerFunction("handle");
    List<String> items = IntStream.range(0, ITEM_COUNT).mapToObj(Integer::toString).collect(toList());
    Pipeline p = Pipeline.create();
    StreamStage<String> stage = p.readFrom(TestSources.items(items)).addTimestamps(x -> 0, 0);
    // When
    StreamStage<String> mapped = stage.apply(mapUsingPython(cfg)).setLocalParallelism(2);
    // Then
    mapped.writeTo(AssertionSinks.assertAnyOrder("Python didn't map the items correctly", items.stream().map(i -> "echo-" + i).collect(toList())));
    instance().getJet().newJob(p).join();
}