Examples with EdgeConfig com.hazelcast.jet.config.EdgeConfig used on opensource projects

Example 6 with EdgeConfig

use of com.hazelcast.jet.config.EdgeConfig in project hazelcast by hazelcast.

the class DotTest method when_dagToDotString.

@Test
public void when_dagToDotString() {
    DAG dag = new DAG();
    Vertex a = dag.newVertex("a", noopP()).localParallelism(1);
    Vertex b = dag.newVertex("b", noopP());
    Vertex c = dag.newVertex("c", noopP()).localParallelism(1);
    Vertex d = dag.newVertex("d", noopP()).localParallelism(1);
    dag.edge(from(a, 0).to(c, 0).partitioned(wholeItem()).setConfig(new EdgeConfig().setQueueSize(128)));
    dag.edge(from(a, 1).to(b, 0).broadcast().distributed());
    String actual = dag.toDotString();
    System.out.println(actual);
    // contains multiple subgraphs, order isn't stable, we'll assert individual lines and the length
    assertTrue(actual.startsWith("digraph DAG {"));
    assertTrue(actual.contains("\"a\" [localParallelism=1];"));
    assertTrue(actual.contains("\"b\" [localParallelism=default];"));
    assertTrue(actual.contains("\"c\" [localParallelism=1];"));
    assertTrue(actual.contains("\"d\" [localParallelism=1];"));
    assertTrue(actual.contains("\"a\" -> \"c\" [label=\"partitioned\", taillabel=0, queueSize=128];"));
    assertTrue(actual.contains("\"a\" -> \"b\" [label=\"distributed-broadcast\", taillabel=1, queueSize=1024]"));
    assertTrue(actual.endsWith("\n}"));
}

Example 7 with EdgeConfig

use of com.hazelcast.jet.config.EdgeConfig in project hazelcast by hazelcast.

the class DotTest method when_dagToDotStringNonDefaults.

@Test
public void when_dagToDotStringNonDefaults() {
    DAG dag = new DAG();
    Vertex a = dag.newVertex("a", noopP()).localParallelism(1);
    Vertex b = dag.newVertex("b", noopP());
    Vertex c = dag.newVertex("c", noopP()).localParallelism(1);
    Vertex d = dag.newVertex("d", noopP()).localParallelism(1);
    dag.edge(from(a, 0).to(c, 0).partitioned(wholeItem()).setConfig(new EdgeConfig().setQueueSize(128)));
    dag.edge(from(a, 1).to(b, 0).broadcast().distributed());
    String actual = dag.toDotString(16, 16384);
    System.out.println(actual);
    // contains multiple subgraphs, order isn't stable, we'll assert individual lines and the length
    assertTrue(actual.startsWith("digraph DAG {"));
    assertTrue(actual.contains("\"a\" [localParallelism=1];"));
    assertTrue(actual.contains("\"b\" [localParallelism=16];"));
    assertTrue(actual.contains("\"c\" [localParallelism=1];"));
    assertTrue(actual.contains("\"d\" [localParallelism=1];"));
    assertTrue(actual.contains("\"a\" -> \"c\" [label=\"partitioned\", taillabel=0, queueSize=128];"));
    assertTrue(actual.contains("\"a\" -> \"b\" [label=\"distributed-broadcast\", taillabel=1, queueSize=16384]"));
    assertTrue(actual.endsWith("\n}"));
}

Example 8 with EdgeConfig

use of com.hazelcast.jet.config.EdgeConfig in project hazelcast by hazelcast.

the class ExecutionPlanBuilder method createExecutionPlans.

@SuppressWarnings("checkstyle:ParameterNumber")
public static Map<MemberInfo, ExecutionPlan> createExecutionPlans(NodeEngineImpl nodeEngine, List<MemberInfo> memberInfos, DAG dag, long jobId, long executionId, JobConfig jobConfig, long lastSnapshotId, boolean isLightJob, Subject subject) {
    final int defaultParallelism = nodeEngine.getConfig().getJetConfig().getCooperativeThreadCount();
    final Map<MemberInfo, int[]> partitionsByMember = getPartitionAssignment(nodeEngine, memberInfos);
    final Map<Address, int[]> partitionsByAddress = partitionsByMember.entrySet().stream().collect(toMap(en -> en.getKey().getAddress(), Entry::getValue));
    final List<Address> addresses = toList(partitionsByMember.keySet(), MemberInfo::getAddress);
    final int clusterSize = partitionsByMember.size();
    final boolean isJobDistributed = clusterSize > 1;
    final EdgeConfig defaultEdgeConfig = nodeEngine.getConfig().getJetConfig().getDefaultEdgeConfig();
    final Map<MemberInfo, ExecutionPlan> plans = new HashMap<>();
    int memberIndex = 0;
    for (MemberInfo member : partitionsByMember.keySet()) {
        plans.put(member, new ExecutionPlan(partitionsByAddress, jobConfig, lastSnapshotId, memberIndex++, clusterSize, isLightJob, subject));
    }
    final Map<String, Integer> vertexIdMap = assignVertexIds(dag);
    for (Entry<String, Integer> entry : vertexIdMap.entrySet()) {
        final Vertex vertex = dag.getVertex(entry.getKey());
        assert vertex != null;
        final ProcessorMetaSupplier metaSupplier = vertex.getMetaSupplier();
        final int vertexId = entry.getValue();
        // The local parallelism determination here is effective only
        // in jobs submitted as DAG. Otherwise, in jobs submitted as
        // pipeline, we are already doing this determination while
        // converting it to DAG and there is no vertex left with LP=-1.
        final int localParallelism = vertex.determineLocalParallelism(defaultParallelism);
        final int totalParallelism = localParallelism * clusterSize;
        final List<EdgeDef> inbound = toEdgeDefs(dag.getInboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getSourceName()), isJobDistributed);
        final List<EdgeDef> outbound = toEdgeDefs(dag.getOutboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getDestName()), isJobDistributed);
        String prefix = prefix(jobConfig.getName(), jobId, vertex.getName(), "#PMS");
        ILogger logger = prefixedLogger(nodeEngine.getLogger(metaSupplier.getClass()), prefix);
        JetServiceBackend jetBackend = nodeEngine.getService(JetServiceBackend.SERVICE_NAME);
        JobClassLoaderService jobClassLoaderService = jetBackend.getJobClassLoaderService();
        ClassLoader processorClassLoader = jobClassLoaderService.getClassLoader(jobId);
        try {
            doWithClassLoader(processorClassLoader, () -> metaSupplier.init(new MetaSupplierCtx(nodeEngine, jobId, executionId, jobConfig, logger, vertex.getName(), localParallelism, totalParallelism, clusterSize, isLightJob, partitionsByAddress, subject, processorClassLoader)));
        } catch (Exception e) {
            throw sneakyThrow(e);
        }
        Function<? super Address, ? extends ProcessorSupplier> procSupplierFn = doWithClassLoader(processorClassLoader, () -> metaSupplier.get(addresses));
        for (Entry<MemberInfo, ExecutionPlan> e : plans.entrySet()) {
            final ProcessorSupplier processorSupplier = doWithClassLoader(processorClassLoader, () -> procSupplierFn.apply(e.getKey().getAddress()));
            if (!isLightJob) {
                // We avoid the check for light jobs - the user will get the error anyway, but maybe with less information.
                // And we can recommend the user to use normal job to have more checks.
                checkSerializable(processorSupplier, "ProcessorSupplier in vertex '" + vertex.getName() + '\'');
            }
            final VertexDef vertexDef = new VertexDef(vertexId, vertex.getName(), processorSupplier, localParallelism);
            vertexDef.addInboundEdges(inbound);
            vertexDef.addOutboundEdges(outbound);
            e.getValue().addVertex(vertexDef);
        }
    }
    return plans;
}

Example 9 with EdgeConfig

use of com.hazelcast.jet.config.EdgeConfig in project hazelcast by hazelcast.

the class TestFullApplicationContext method testJetConfig.

@Test
public void testJetConfig() {
    JetConfig jetConfig = config.getJetConfig();
    assertTrue(jetConfig.isEnabled());
    assertTrue(jetConfig.isResourceUploadEnabled());
    assertEquals(4, jetConfig.getCooperativeThreadCount());
    assertEquals(2, jetConfig.getBackupCount());
    assertEquals(200, jetConfig.getFlowControlPeriodMs());
    assertEquals(20000, jetConfig.getScaleUpDelayMillis());
    assertEquals(1000000, jetConfig.getMaxProcessorAccumulatedRecords());
    assertFalse(jetConfig.isLosslessRestartEnabled());
    EdgeConfig edgeConfig = jetConfig.getDefaultEdgeConfig();
    assertEquals(2048, edgeConfig.getQueueSize());
    assertEquals(15000, edgeConfig.getPacketSizeLimit());
    assertEquals(4, edgeConfig.getReceiveWindowMultiplier());
}

Example 10 with EdgeConfig

use of com.hazelcast.jet.config.EdgeConfig in project hazelcast by hazelcast.

the class JobSubmissionSlownessRegressionTest method twoVertex.

private static DAG twoVertex() {
    DAG dag = new DAG();
    Vertex v1 = dag.newVertex("v", noopP());
    Vertex v2 = dag.newVertex("v2", noopP());
    dag.edge(Edge.between(v1, v2).setConfig(new EdgeConfig().setQueueSize(1)));
    return dag;
}