Examples with Vertex com.hazelcast.jet.core.Vertex used on opensource projects

Example 86 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class HazelcastConnectorTest method when_streamMap_withProjectionToNull_then_nullsSkipped.

@Test
public void when_streamMap_withProjectionToNull_then_nullsSkipped() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", SourceProcessors.streamMapP(streamSourceName, mapPutEvents(), (EventJournalMapEvent<Integer, Entry<Integer, String>> entry) -> entry.getNewValue().getValue(), START_FROM_OLDEST, noEventTime()));
    Vertex sink = dag.newVertex("sink", writeListP(streamSinkName));
    dag.edge(between(source, sink));
    Job job = instance().getJet().newJob(dag);
    IMap<Integer, Entry<Integer, String>> sourceMap = instance().getMap(streamSourceName);
    range(0, ENTRY_COUNT).forEach(i -> sourceMap.put(i, entry(i, i % 2 == 0 ? null : String.valueOf(i))));
    assertTrueEventually(() -> checkContents_projectedToNull(streamSinkName));
    job.cancel();
}

Example 87 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class StreamEventJournalPTest method when_processorsWithNoPartitions_then_snapshotRestoreWorks.

@Test
public void when_processorsWithNoPartitions_then_snapshotRestoreWorks() {
    DAG dag = new DAG();
    Vertex vertex = dag.newVertex("src", streamMapP(map.getName(), JournalInitialPosition.START_FROM_OLDEST, noEventTime())).localParallelism(8);
    int partitionCount = instance.getPartitionService().getPartitions().size();
    assertTrue("partition count should be lower than local parallelism", vertex.getLocalParallelism() > partitionCount);
    Job job = instance.getJet().newJob(dag, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(200_000));
    assertJobStatusEventually(job, JobStatus.RUNNING, 25);
    job.restart();
    // Then
    // The job should be running: this test checks that state restored to NoopP, which is
    // created by the meta supplier for processor with no partitions, is ignored.
    sleepMillis(3000);
    assertJobStatusEventually(job, JobStatus.RUNNING, 10);
}

Example 88 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class BatchStageTest method addTimestamps_when_upstreamHasPreferredLocalParallelism_then_lpMatchUpstream.

@Test
public void addTimestamps_when_upstreamHasPreferredLocalParallelism_then_lpMatchUpstream() {
    // Given
    int lp = 11;
    // When
    p.readFrom(Sources.batchFromProcessor("src", ProcessorMetaSupplier.of(lp, ProcessorSupplier.of(noopP())))).addTimestamps(o -> 0L, 0).writeTo(Sinks.noop());
    DAG dag = p.toDag();
    // Then
    Vertex tsVertex = dag.getVertex("add-timestamps");
    assertEquals(lp, tsVertex.getLocalParallelism());
}

Example 89 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class LightJobTest method smokeTest_dag.

@Test
public void smokeTest_dag() {
    List<Integer> items = IntStream.range(0, 1_000).boxed().collect(Collectors.toList());
    DAG dag = new DAG();
    Vertex src = dag.newVertex("src", processorFromPipelineSource(TestSources.items(items)));
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP("sink"));
    dag.edge(between(src, sink).distributed());
    submittingInstance().getJet().newLightJob(dag).join();
    List<Integer> result = instance().getList("sink");
    assertThat(result).containsExactlyInAnyOrderElementsOf(items);
}

Example 90 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class WordCountTest method generateMockInput.

private void generateMockInput() {
    logger.info("Generating input");
    final DAG dag = new DAG();
    Vertex source = dag.newVertex("source", (List<Address> addrs) -> (Address addr) -> ProcessorSupplier.of(addr.equals(addrs.get(0)) ? MockInputP::new : noopP()));
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeMapP("words"));
    dag.edge(between(source.localParallelism(1), sink.localParallelism(1)));
    instance.getJet().newJob(dag).join();
    logger.info("Input generated.");
}