Examples with ProcessorSupplier com.hazelcast.jet.core.ProcessorSupplier used on opensource projects

Example 11 with ProcessorSupplier

use of com.hazelcast.jet.core.ProcessorSupplier in project hazelcast by hazelcast.

the class AsyncTransformUsingServiceBatchP_IntegrationTest method stressTestInt.

private void stressTestInt(boolean restart) {
    /*
        This is a stress test of the cooperative emission using the DAG api. Only through DAG
        API we can configure edge queue sizes, which we use to cause more trouble for the
        cooperative emission.
         */
    // add more input to the source map
    int numItems = 10_000;
    journaledMap.putAll(IntStream.range(NUM_ITEMS, numItems).boxed().collect(toMap(i -> i, i -> i)));
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", throttle(streamMapP(journaledMap.getName(), alwaysTrue(), EventJournalMapEvent::getNewValue, START_FROM_OLDEST, eventTimePolicy(i -> (long) ((Integer) i), WatermarkPolicy.limitingLag(10), 10, 0, 0)), 5000));
    BiFunctionEx<ExecutorService, List<Integer>, CompletableFuture<Traverser<String>>> flatMapAsyncFn = transformNotPartitionedFn(i -> traverseItems(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5")).andThen(r -> r.thenApply(results -> traverseIterable(results).flatMap(Function.identity())));
    ProcessorSupplier processorSupplier = AsyncTransformUsingServiceBatchedP.supplier(serviceFactory, DEFAULT_MAX_CONCURRENT_OPS, 128, flatMapAsyncFn);
    Vertex map = dag.newVertex("map", processorSupplier).localParallelism(2);
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP(sinkList.getName()));
    // Use a shorter queue to not block the barrier from the source for too long due to
    // the backpressure from the slow mapper
    EdgeConfig edgeToMapperConfig = new EdgeConfig().setQueueSize(128);
    // Use a shorter queue on output from the mapper so that we experience backpressure
    // from the sink
    EdgeConfig edgeFromMapperConfig = new EdgeConfig().setQueueSize(10);
    dag.edge(between(source, map).setConfig(edgeToMapperConfig)).edge(between(map, sink).setConfig(edgeFromMapperConfig));
    Job job = instance().getJet().newJob(dag, jobConfig);
    for (int i = 0; restart && i < 5; i++) {
        assertNotNull(job);
        assertTrueEventually(() -> {
            JobStatus status = job.getStatus();
            assertTrue("status=" + status, status == RUNNING || status == COMPLETED);
        });
        sleepMillis(100);
        try {
            job.restart();
        } catch (IllegalStateException e) {
            assertTrue(e.toString(), e.getMessage().startsWith("Cannot RESTART_GRACEFUL"));
            break;
        }
    }
    assertResult(i -> Stream.of(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5"), numItems);
}

Example 12 with ProcessorSupplier

use of com.hazelcast.jet.core.ProcessorSupplier in project hazelcast by hazelcast.

the class AsyncTransformUsingServiceP_IntegrationTest method stressTestInt.

private void stressTestInt(boolean restart) {
    /*
        This is a stress test of the cooperative emission using the DAG api. Only through DAG
        API we can configure edge queue sizes, which we use to cause more trouble for the
        cooperative emission.
         */
    // add more input to the source map
    int numItems = 10_000;
    journaledMap.putAll(IntStream.range(NUM_ITEMS, numItems).boxed().collect(toMap(i -> i, i -> i)));
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", throttle(streamMapP(journaledMap.getName(), alwaysTrue(), EventJournalMapEvent::getNewValue, START_FROM_OLDEST, eventTimePolicy(i -> (long) ((Integer) i), WatermarkPolicy.limitingLag(10), 10, 0, 0)), 5000));
    BiFunctionEx<ExecutorService, Integer, CompletableFuture<Traverser<String>>> flatMapAsyncFn = transformNotPartitionedFn(i -> traverseItems(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5"));
    ProcessorSupplier processorSupplier = ordered ? AsyncTransformUsingServiceOrderedP.supplier(serviceFactory, DEFAULT_MAX_CONCURRENT_OPS, flatMapAsyncFn) : AsyncTransformUsingServiceUnorderedP.supplier(serviceFactory, DEFAULT_MAX_CONCURRENT_OPS, flatMapAsyncFn, identity());
    Vertex map = dag.newVertex("map", processorSupplier).localParallelism(2);
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP(sinkList.getName()));
    // Use a shorter queue to not block the barrier from the source for too long due to
    // the backpressure from the slow mapper
    EdgeConfig edgeToMapperConfig = new EdgeConfig().setQueueSize(128);
    // Use a shorter queue on output from the mapper so that we experience backpressure
    // from the sink
    EdgeConfig edgeFromMapperConfig = new EdgeConfig().setQueueSize(10);
    dag.edge(between(source, map).setConfig(edgeToMapperConfig)).edge(between(map, sink).setConfig(edgeFromMapperConfig));
    Job job = instance().getJet().newJob(dag, jobConfig);
    for (int i = 0; restart && i < 5; i++) {
        assertJobStatusEventually(job, RUNNING);
        sleepMillis(100);
        job.restart();
    }
    assertResultEventually(i -> Stream.of(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5"), numItems);
}

Example 13 with ProcessorSupplier

use of com.hazelcast.jet.core.ProcessorSupplier in project hazelcast by hazelcast.

the class StreamFilesPTest method when_metaSupplier_then_returnsCorrectProcessors.

@Test
public void when_metaSupplier_then_returnsCorrectProcessors() throws Exception {
    ProcessorMetaSupplier metaSupplier = streamFilesP(workDir.getAbsolutePath(), UTF_8, "*", false, Util::entry);
    Address a = new Address();
    ProcessorSupplier supplier = metaSupplier.get(singletonList(a)).apply(a);
    supplier.init(new TestProcessorContext());
    assertEquals(1, supplier.get(1).size());
    supplier.close(null);
}

Example 14 with ProcessorSupplier

use of com.hazelcast.jet.core.ProcessorSupplier in project gora by apache.

the class SinkProcessor method get.

@Nonnull
@Override
public Function<? super Address, ? extends ProcessorSupplier> get(@Nonnull List<Address> addresses) {
    Map<Address, ProcessorSupplier> map = new HashMap<>();
    for (int i = 0; i < addresses.size(); i++) {
        // globalIndexBase is the first processor index in a certain Jet-Cluster member
        int globalIndexBase = localParallelism * i;
        // processorCount will be equal to localParallelism:
        ProcessorSupplier supplier = processorCount -> range(globalIndexBase, globalIndexBase + processorCount).mapToObj(globalIndex -> new SinkProcessor<KeyOut, ValueOut>()).collect(toList());
        map.put(addresses.get(i), supplier);
    }
    return map::get;
}

Example 15 with ProcessorSupplier

use of com.hazelcast.jet.core.ProcessorSupplier in project gora by apache.

the class GoraJetProcessor method get.

@Nonnull
@Override
public Function<? super Address, ? extends ProcessorSupplier> get(@Nonnull List<Address> addresses) {
    Map<Address, ProcessorSupplier> map = new HashMap<>();
    for (int i = 0; i < addresses.size(); i++) {
        // We'll calculate the global index of each processor in the cluster:
        // globalIndexBase is the first processor index in a certain Jet-Cluster member
        int globalIndexBase = localParallelism * i;
        // processorCount will be equal to localParallelism:
        ProcessorSupplier supplier = processorCount -> range(globalIndexBase, globalIndexBase + processorCount).mapToObj(globalIndex -> new GoraJetProcessor<KeyIn, ValueIn>(getPartitionedData(globalIndex))).collect(toList());
        map.put(addresses.get(i), supplier);
    }
    return map::get;
}