Examples with EdgeConfig com.hazelcast.jet.config.EdgeConfig used on opensource projects

Example 1 with EdgeConfig

use of com.hazelcast.jet.config.EdgeConfig in project hazelcast-jet by hazelcast.

the class TestApplicationContext method assertJetConfig.

private void assertJetConfig() {
    JetConfig jetConfig = jetInstance.getConfig();
    Config hazelcastConfig = jetConfig.getHazelcastConfig();
    assertHazelcastConfig(hazelcastConfig);
    InstanceConfig instanceConfig = jetConfig.getInstanceConfig();
    assertEquals(4, instanceConfig.getBackupCount());
    assertEquals(2, instanceConfig.getCooperativeThreadCount());
    assertEquals(200, instanceConfig.getFlowControlPeriodMs());
    assertEquals("springTempDir", instanceConfig.getTempDir());
    EdgeConfig edgeConfig = jetConfig.getDefaultEdgeConfig();
    assertEquals(8, edgeConfig.getQueueSize());
    assertEquals(3, edgeConfig.getPacketSizeLimit());
    assertEquals(5, edgeConfig.getReceiveWindowMultiplier());
    assertEquals("bar", jetConfig.getProperties().getProperty("foo"));
}

Example 2 with EdgeConfig

use of com.hazelcast.jet.config.EdgeConfig in project hazelcast by hazelcast.

the class Util method copyMapUsingJob.

// used in jet-enterprise
@SuppressWarnings("WeakerAccess")
public static CompletableFuture<Void> copyMapUsingJob(HazelcastInstance instance, int queueSize, String sourceMap, String targetMap) {
    DAG dag = new DAG();
    Vertex source = dag.newVertex("readMap(" + sourceMap + ')', readMapP(sourceMap));
    Vertex sink = dag.newVertex("writeMap(" + targetMap + ')', writeMapP(targetMap));
    dag.edge(between(source, sink).setConfig(new EdgeConfig().setQueueSize(queueSize)));
    JobConfig jobConfig = new JobConfig().setName("copy-" + sourceMap + "-to-" + targetMap);
    return instance.getJet().newJob(dag, jobConfig).getFuture();
}

Example 3 with EdgeConfig

use of com.hazelcast.jet.config.EdgeConfig in project hazelcast by hazelcast.

the class ConfigXmlGenerator method jetConfig.

private static void jetConfig(XmlGenerator gen, Config config) {
    JetConfig jetConfig = config.getJetConfig();
    EdgeConfig edgeConfig = jetConfig.getDefaultEdgeConfig();
    gen.open("jet", "enabled", jetConfig.isEnabled(), "resource-upload-enabled", jetConfig.isResourceUploadEnabled()).node("cooperative-thread-count", jetConfig.getCooperativeThreadCount()).node("flow-control-period", jetConfig.getFlowControlPeriodMs()).node("backup-count", jetConfig.getBackupCount()).node("scale-up-delay-millis", jetConfig.getScaleUpDelayMillis()).node("lossless-restart-enabled", jetConfig.isLosslessRestartEnabled()).node("max-processor-accumulated-records", jetConfig.getMaxProcessorAccumulatedRecords()).open("edge-defaults").node("queue-size", edgeConfig.getQueueSize()).node("packet-size-limit", edgeConfig.getPacketSizeLimit()).node("receive-window-multiplier", edgeConfig.getReceiveWindowMultiplier()).close().close();
}

Example 4 with EdgeConfig

use of com.hazelcast.jet.config.EdgeConfig in project hazelcast by hazelcast.

the class AsyncTransformUsingServiceBatchP_IntegrationTest method stressTestInt.

private void stressTestInt(boolean restart) {
    /*
        This is a stress test of the cooperative emission using the DAG api. Only through DAG
        API we can configure edge queue sizes, which we use to cause more trouble for the
        cooperative emission.
         */
    // add more input to the source map
    int numItems = 10_000;
    journaledMap.putAll(IntStream.range(NUM_ITEMS, numItems).boxed().collect(toMap(i -> i, i -> i)));
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", throttle(streamMapP(journaledMap.getName(), alwaysTrue(), EventJournalMapEvent::getNewValue, START_FROM_OLDEST, eventTimePolicy(i -> (long) ((Integer) i), WatermarkPolicy.limitingLag(10), 10, 0, 0)), 5000));
    BiFunctionEx<ExecutorService, List<Integer>, CompletableFuture<Traverser<String>>> flatMapAsyncFn = transformNotPartitionedFn(i -> traverseItems(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5")).andThen(r -> r.thenApply(results -> traverseIterable(results).flatMap(Function.identity())));
    ProcessorSupplier processorSupplier = AsyncTransformUsingServiceBatchedP.supplier(serviceFactory, DEFAULT_MAX_CONCURRENT_OPS, 128, flatMapAsyncFn);
    Vertex map = dag.newVertex("map", processorSupplier).localParallelism(2);
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP(sinkList.getName()));
    // Use a shorter queue to not block the barrier from the source for too long due to
    // the backpressure from the slow mapper
    EdgeConfig edgeToMapperConfig = new EdgeConfig().setQueueSize(128);
    // Use a shorter queue on output from the mapper so that we experience backpressure
    // from the sink
    EdgeConfig edgeFromMapperConfig = new EdgeConfig().setQueueSize(10);
    dag.edge(between(source, map).setConfig(edgeToMapperConfig)).edge(between(map, sink).setConfig(edgeFromMapperConfig));
    Job job = instance().getJet().newJob(dag, jobConfig);
    for (int i = 0; restart && i < 5; i++) {
        assertNotNull(job);
        assertTrueEventually(() -> {
            JobStatus status = job.getStatus();
            assertTrue("status=" + status, status == RUNNING || status == COMPLETED);
        });
        sleepMillis(100);
        try {
            job.restart();
        } catch (IllegalStateException e) {
            assertTrue(e.toString(), e.getMessage().startsWith("Cannot RESTART_GRACEFUL"));
            break;
        }
    }
    assertResult(i -> Stream.of(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5"), numItems);
}

Example 5 with EdgeConfig

use of com.hazelcast.jet.config.EdgeConfig in project hazelcast by hazelcast.

the class AsyncTransformUsingServiceP_IntegrationTest method stressTestInt.

private void stressTestInt(boolean restart) {
    /*
        This is a stress test of the cooperative emission using the DAG api. Only through DAG
        API we can configure edge queue sizes, which we use to cause more trouble for the
        cooperative emission.
         */
    // add more input to the source map
    int numItems = 10_000;
    journaledMap.putAll(IntStream.range(NUM_ITEMS, numItems).boxed().collect(toMap(i -> i, i -> i)));
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", throttle(streamMapP(journaledMap.getName(), alwaysTrue(), EventJournalMapEvent::getNewValue, START_FROM_OLDEST, eventTimePolicy(i -> (long) ((Integer) i), WatermarkPolicy.limitingLag(10), 10, 0, 0)), 5000));
    BiFunctionEx<ExecutorService, Integer, CompletableFuture<Traverser<String>>> flatMapAsyncFn = transformNotPartitionedFn(i -> traverseItems(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5"));
    ProcessorSupplier processorSupplier = ordered ? AsyncTransformUsingServiceOrderedP.supplier(serviceFactory, DEFAULT_MAX_CONCURRENT_OPS, flatMapAsyncFn) : AsyncTransformUsingServiceUnorderedP.supplier(serviceFactory, DEFAULT_MAX_CONCURRENT_OPS, flatMapAsyncFn, identity());
    Vertex map = dag.newVertex("map", processorSupplier).localParallelism(2);
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP(sinkList.getName()));
    // Use a shorter queue to not block the barrier from the source for too long due to
    // the backpressure from the slow mapper
    EdgeConfig edgeToMapperConfig = new EdgeConfig().setQueueSize(128);
    // Use a shorter queue on output from the mapper so that we experience backpressure
    // from the sink
    EdgeConfig edgeFromMapperConfig = new EdgeConfig().setQueueSize(10);
    dag.edge(between(source, map).setConfig(edgeToMapperConfig)).edge(between(map, sink).setConfig(edgeFromMapperConfig));
    Job job = instance().getJet().newJob(dag, jobConfig);
    for (int i = 0; restart && i < 5; i++) {
        assertJobStatusEventually(job, RUNNING);
        sleepMillis(100);
        job.restart();
    }
    assertResultEventually(i -> Stream.of(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5"), numItems);
}