Examples with Vertex com.hazelcast.jet.core.Vertex used on opensource projects

Example 61 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class JobMetrics_MiscTest method when_jobSuspended_andMetricsNotStored_then_onlyPeriodicMetricsReturned.

@Test
public void when_jobSuspended_andMetricsNotStored_then_onlyPeriodicMetricsReturned() throws Throwable {
    DAG dag = new DAG();
    Vertex v1 = dag.newVertex("v1", TestProcessors.MockP::new);
    Vertex v2 = dag.newVertex("v2", (SupplierEx<Processor>) TestProcessors.NoOutputSourceP::new);
    dag.edge(between(v1, v2));
    // init
    JobConfig config = new JobConfig().setMetricsEnabled(// enable metric collection
    true).setStoreMetricsAfterJobCompletion(// disable metric saving on completion
    false);
    Job job = hz().getJet().newJob(dag, config);
    // when
    TestProcessors.NoOutputSourceP.executionStarted.await();
    // then
    assertJobStatusEventually(job, JobStatus.RUNNING);
    assertTrueEventually(() -> assertJobHasMetrics(job, false));
    // when
    job.suspend();
    // then
    assertJobStatusEventually(job, SUSPENDED);
    assertTrueEventually(() -> assertEmptyJobMetrics(job, false));
    // when
    job.resume();
    // then
    assertJobStatusEventually(job, RUNNING);
    assertTrueEventually(() -> assertJobHasMetrics(job, false));
    // when
    TestProcessors.NoOutputSourceP.proceedLatch.countDown();
    job.join();
    // then
    assertJobStatusEventually(job, JobStatus.COMPLETED);
    assertEmptyJobMetrics(job, false);
}

Example 62 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class MetricsTest method nonCooperativeProcessor.

@Test
public void nonCooperativeProcessor() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", TestProcessors.ListSource.supplier(asList(1L, 2L, 3L)));
    Vertex map = dag.newVertex("map", new NonCoopTransformPSupplier((FunctionEx<Long, Long>) l -> {
        Metrics.metric("mapped").increment();
        return l * 10L;
    }));
    Vertex sink = dag.newVertex("sink", writeListP("results"));
    dag.edge(between(source, map)).edge(between(map, sink));
    Job job = runPipeline(dag);
    JobMetricsChecker checker = new JobMetricsChecker(job);
    checker.assertSummedMetricValue("mapped", 3L);
    assertEquals(new HashSet<>(Arrays.asList(10L, 20L, 30L)), new HashSet<>(instance.getList("results")));
}

Example 63 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class AsyncTransformUsingServiceBatchP_IntegrationTest method stressTestInt.

private void stressTestInt(boolean restart) {
    /*
        This is a stress test of the cooperative emission using the DAG api. Only through DAG
        API we can configure edge queue sizes, which we use to cause more trouble for the
        cooperative emission.
         */
    // add more input to the source map
    int numItems = 10_000;
    journaledMap.putAll(IntStream.range(NUM_ITEMS, numItems).boxed().collect(toMap(i -> i, i -> i)));
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", throttle(streamMapP(journaledMap.getName(), alwaysTrue(), EventJournalMapEvent::getNewValue, START_FROM_OLDEST, eventTimePolicy(i -> (long) ((Integer) i), WatermarkPolicy.limitingLag(10), 10, 0, 0)), 5000));
    BiFunctionEx<ExecutorService, List<Integer>, CompletableFuture<Traverser<String>>> flatMapAsyncFn = transformNotPartitionedFn(i -> traverseItems(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5")).andThen(r -> r.thenApply(results -> traverseIterable(results).flatMap(Function.identity())));
    ProcessorSupplier processorSupplier = AsyncTransformUsingServiceBatchedP.supplier(serviceFactory, DEFAULT_MAX_CONCURRENT_OPS, 128, flatMapAsyncFn);
    Vertex map = dag.newVertex("map", processorSupplier).localParallelism(2);
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP(sinkList.getName()));
    // Use a shorter queue to not block the barrier from the source for too long due to
    // the backpressure from the slow mapper
    EdgeConfig edgeToMapperConfig = new EdgeConfig().setQueueSize(128);
    // Use a shorter queue on output from the mapper so that we experience backpressure
    // from the sink
    EdgeConfig edgeFromMapperConfig = new EdgeConfig().setQueueSize(10);
    dag.edge(between(source, map).setConfig(edgeToMapperConfig)).edge(between(map, sink).setConfig(edgeFromMapperConfig));
    Job job = instance().getJet().newJob(dag, jobConfig);
    for (int i = 0; restart && i < 5; i++) {
        assertNotNull(job);
        assertTrueEventually(() -> {
            JobStatus status = job.getStatus();
            assertTrue("status=" + status, status == RUNNING || status == COMPLETED);
        });
        sleepMillis(100);
        try {
            job.restart();
        } catch (IllegalStateException e) {
            assertTrue(e.toString(), e.getMessage().startsWith("Cannot RESTART_GRACEFUL"));
            break;
        }
    }
    assertResult(i -> Stream.of(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5"), numItems);
}

Example 64 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class HazelcastRemoteConnectorTest method when_streamRemoteMap.

@Test
public void when_streamRemoteMap() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex(SOURCE_NAME, streamRemoteMapP(SOURCE_NAME, clientConfig, START_FROM_OLDEST, eventTimePolicy(Entry<Integer, Integer>::getValue, limitingLag(0), 1, 0, 10_000)));
    Vertex sink = dag.newVertex(SINK_NAME, writeListP(SINK_NAME));
    dag.edge(between(source, sink));
    Job job = localHz.getJet().newJob(dag);
    populateMap(remoteHz.getMap(SOURCE_NAME));
    assertSizeEventually(ITEM_COUNT, localHz.getList(SINK_NAME));
    job.cancel();
}

Example 65 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class HazelcastRemoteConnectorTest method when_readRemoteMap_withPredicateAndFunction.

@Test
public void when_readRemoteMap_withPredicateAndFunction() {
    populateMap(remoteHz.getMap(SOURCE_NAME));
    DAG dag = new DAG();
    Vertex source = dag.newVertex(SOURCE_NAME, readRemoteMapP(SOURCE_NAME, clientConfig, e -> !e.getKey().equals(0), Entry::getValue)).localParallelism(4);
    Vertex sink = dag.newVertex(SINK_NAME, writeListP(SINK_NAME)).localParallelism(1);
    dag.edge(between(source, sink));
    executeAndWait(dag);
    IList<Object> list = localHz.getList(SINK_NAME);
    assertEquals(ITEM_COUNT - 1, list.size());
    assertFalse(list.contains(0));
    assertTrue(list.contains(1));
}