Examples with ListSource com.hazelcast.jet.core.TestProcessors.ListSource used on opensource projects

Example 1 with ListSource

use of com.hazelcast.jet.core.TestProcessors.ListSource in project hazelcast-jet by hazelcast.

the class WatermarkMaxRetention_IntegrationTest method test_onEdgeCoalescing.

@Test
public void test_onEdgeCoalescing() {
    DAG dag = new DAG();
    // a vertex with two processor instances, one will emit a wm and the other won't
    Vertex source = dag.newVertex("source", (int count) -> asList(new ListSource(singletonList(new Watermark(1))), new StuckForeverSourceP())).localParallelism(2);
    Vertex map = dag.newVertex("map", MapWmToStringP::new).localParallelism(1);
    Vertex sink = dag.newVertex("sink", writeListP(SINK_NAME));
    dag.edge(between(source, map)).edge(between(map, sink));
    doTest(dag);
}

Example 2 with ListSource

use of com.hazelcast.jet.core.TestProcessors.ListSource in project hazelcast-jet by hazelcast.

the class WatermarkMaxRetention_IntegrationTest method test_onVertexCoalescing.

@Test
public void test_onVertexCoalescing() {
    DAG dag = new DAG();
    Vertex source0 = dag.newVertex("source0", () -> new ListSource(singletonList(new Watermark(1)))).localParallelism(1);
    Vertex source1 = dag.newVertex("source1", StuckForeverSourceP::new);
    // a vertex with two inputs, one will emit a wm and the other won't
    Vertex map = dag.newVertex("map", MapWmToStringP::new).localParallelism(1);
    Vertex sink = dag.newVertex("sink", writeListP(SINK_NAME));
    dag.edge(from(source0).to(map, 0)).edge(from(source1).to(map, 1)).edge(between(map, sink));
    doTest(dag);
}

Example 3 with ListSource

use of com.hazelcast.jet.core.TestProcessors.ListSource in project hazelcast by hazelcast.

the class Processors_globalAggregationIntegrationTest method runTest.

private void runTest(List<Long> sourceItems, Long expectedOutput) throws Exception {
    HazelcastInstance instance = createHazelcastInstance();
    AggregateOperation1<Long, ?, Long> summingOp = summingLong((Long l) -> l);
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", () -> new ListSource(sourceItems)).localParallelism(1);
    Vertex sink = dag.newVertex("sink", writeListP("sink"));
    if (singleStageProcessor) {
        Vertex aggregate = dag.newVertex("aggregate", Processors.aggregateP(summingOp)).localParallelism(1);
        dag.edge(between(source, aggregate).distributed().allToOne("foo")).edge(between(aggregate, sink).isolated());
    } else {
        Vertex accumulate = dag.newVertex("accumulate", Processors.accumulateP(summingOp));
        Vertex combine = dag.newVertex("combine", combineP(summingOp)).localParallelism(1);
        dag.edge(between(source, accumulate)).edge(between(accumulate, combine).distributed().allToOne("foo")).edge(between(combine, sink).isolated());
    }
    instance.getJet().newJob(dag).join();
    IList<Long> sinkList = instance.getList("sink");
    assertEquals(singletonList(expectedOutput), new ArrayList<>(sinkList));
    // wait a little more and make sure, that there are no more frames
    Thread.sleep(1000);
    assertEquals(singletonList(expectedOutput), new ArrayList<>(sinkList));
    assertEquals(expectedOutput, sinkList.get(0));
}

Example 4 with ListSource

use of com.hazelcast.jet.core.TestProcessors.ListSource in project hazelcast by hazelcast.

the class ExecutionLifecycle_RestartableExceptionTest method when_inProcessor.

private void when_inProcessor(SupplierEx<Processor> supplier) {
    Vertex src = dag.newVertex("src", () -> new ListSource(1));
    Vertex v = dag.newVertex("v", new MockPS(supplier, MEMBER_COUNT));
    dag.edge(between(src, v));
    Job job = newJob(dag);
    if (useLightJob) {
        assertThatThrownBy(() -> job.join()).hasRootCause(RESTARTABLE_EXCEPTION);
    } else {
        assertTrueEventually(() -> assertGreaterOrEquals("MockPS.init not call count", MockPS.initCount.get(), 2 * MEMBER_COUNT), 10);
    }
}

Example 5 with ListSource

use of com.hazelcast.jet.core.TestProcessors.ListSource in project hazelcast by hazelcast.

the class JobTest method when_serializerIsRegistered_then_itIsAvailableForTheJob.

@Test
public void when_serializerIsRegistered_then_itIsAvailableForTheJob() {
    // Given
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", () -> new ListSource(new Value(1), new Value(2)));
    Vertex sink = dag.newVertex("sink", DiagnosticProcessors.writeLoggerP());
    dag.edge(between(source, sink).distributed().partitioned(FunctionEx.identity()));
    JobConfig config = new JobConfig().registerSerializer(Value.class, ValueSerializer.class);
    // When
    Job job = instance().getJet().newJob(dag, config);
    // Then
    assertJobStatusEventually(job, COMPLETED);
}