Examples with ProcessorSupplier com.hazelcast.jet.core.ProcessorSupplier used on opensource projects

Example 16 with ProcessorSupplier

use of com.hazelcast.jet.core.ProcessorSupplier in project hazelcast-jet by hazelcast.

the class ExecutionPlanBuilder method createExecutionPlans.

public static Map<MemberInfo, ExecutionPlan> createExecutionPlans(NodeEngine nodeEngine, MembersView membersView, DAG dag, JobConfig jobConfig, long lastSnapshotId) {
    final JetInstance instance = getJetInstance(nodeEngine);
    final int defaultParallelism = instance.getConfig().getInstanceConfig().getCooperativeThreadCount();
    final Collection<MemberInfo> members = new HashSet<>(membersView.size());
    final Address[] partitionOwners = new Address[nodeEngine.getPartitionService().getPartitionCount()];
    initPartitionOwnersAndMembers(nodeEngine, membersView, members, partitionOwners);
    final List<Address> addresses = members.stream().map(MemberInfo::getAddress).collect(toList());
    final int clusterSize = members.size();
    final boolean isJobDistributed = clusterSize > 1;
    final EdgeConfig defaultEdgeConfig = instance.getConfig().getDefaultEdgeConfig();
    final Map<MemberInfo, ExecutionPlan> plans = members.stream().collect(toMap(m -> m, m -> new ExecutionPlan(partitionOwners, jobConfig, lastSnapshotId)));
    final Map<String, Integer> vertexIdMap = assignVertexIds(dag);
    for (Entry<String, Integer> entry : vertexIdMap.entrySet()) {
        final Vertex vertex = dag.getVertex(entry.getKey());
        final ProcessorMetaSupplier metaSupplier = vertex.getMetaSupplier();
        final int vertexId = entry.getValue();
        final int localParallelism = determineParallelism(vertex, metaSupplier.preferredLocalParallelism(), defaultParallelism);
        final int totalParallelism = localParallelism * clusterSize;
        final List<EdgeDef> inbound = toEdgeDefs(dag.getInboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getSourceName()), isJobDistributed);
        final List<EdgeDef> outbound = toEdgeDefs(dag.getOutboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getDestName()), isJobDistributed);
        final ILogger logger = nodeEngine.getLogger(String.format("%s.%s#ProcessorMetaSupplier", metaSupplier.getClass().getName(), vertex.getName()));
        metaSupplier.init(new MetaSupplierCtx(instance, logger, vertex.getName(), localParallelism, totalParallelism));
        Function<Address, ProcessorSupplier> procSupplierFn = metaSupplier.get(addresses);
        int procIdxOffset = 0;
        for (Entry<MemberInfo, ExecutionPlan> e : plans.entrySet()) {
            final ProcessorSupplier processorSupplier = procSupplierFn.apply(e.getKey().getAddress());
            checkSerializable(processorSupplier, "ProcessorSupplier in vertex '" + vertex.getName() + '\'');
            final VertexDef vertexDef = new VertexDef(vertexId, vertex.getName(), processorSupplier, procIdxOffset, localParallelism, totalParallelism);
            vertexDef.addInboundEdges(inbound);
            vertexDef.addOutboundEdges(outbound);
            e.getValue().addVertex(vertexDef);
            procIdxOffset += localParallelism;
        }
    }
    return plans;
}

Example 17 with ProcessorSupplier

use of com.hazelcast.jet.core.ProcessorSupplier in project hazelcast-jet by hazelcast.

the class StreamFilesPTest method when_metaSupplier_then_returnsCorrectProcessors.

@Test
public void when_metaSupplier_then_returnsCorrectProcessors() {
    ProcessorMetaSupplier metaSupplier = streamFilesP(workDir.getAbsolutePath(), UTF_8, "*", Util::entry);
    Address a = new Address();
    ProcessorSupplier supplier = metaSupplier.get(singletonList(a)).apply(a);
    supplier.init(new TestProcessorContext());
    assertEquals(1, supplier.get(1).size());
    supplier.close(null);
}

Example 18 with ProcessorSupplier

use of com.hazelcast.jet.core.ProcessorSupplier in project hazelcast-jet by hazelcast.

the class TestSupportTest method when_processorSupplierTested_then_completeCalled.

@Test
public void when_processorSupplierTested_then_completeCalled() {
    boolean[] completeCalled = { false };
    ProcessorSupplier supplier = new ProcessorSupplier() {

        @Nonnull
        @Override
        public Collection<? extends Processor> get(int count) {
            assertEquals(1, count);
            return singletonList(noopP().get());
        }

        @Override
        public void close(Throwable error) {
            completeCalled[0] = true;
        }
    };
    TestSupport.verifyProcessor(supplier).expectOutput(emptyList());
    assertTrue("PS.complete not called", completeCalled[0]);
    // test once more with PMS
    completeCalled[0] = false;
    TestSupport.verifyProcessor(ProcessorMetaSupplier.of(supplier)).expectOutput(emptyList());
    assertTrue("PS.complete not called when using PMS", completeCalled[0]);
}

Example 19 with ProcessorSupplier

use of com.hazelcast.jet.core.ProcessorSupplier in project hazelcast-jet-reference-manual by hazelcast.

the class S8 method get.

@Override
@Nonnull
public Function<Address, ProcessorSupplier> get(@Nonnull List<Address> addresses) {
    Map<Address, ProcessorSupplier> map = new HashMap<>();
    for (int i = 0; i < addresses.size(); i++) {
        // We'll calculate the global index of each processor in the cluster:
        int globalIndexBase = localParallelism * i;
        // Capture the value of the transient field for the lambdas below:
        int divisor = totalParallelism;
        // processorCount will be equal to localParallelism:
        ProcessorSupplier supplier = processorCount -> range(globalIndexBase, globalIndexBase + processorCount).mapToObj(globalIndex -> new GenerateNumbersP(upperBound, divisor, globalIndex)).collect(toList());
        map.put(addresses.get(i), supplier);
    }
    return map::get;
}

Example 20 with ProcessorSupplier

use of com.hazelcast.jet.core.ProcessorSupplier in project hazelcast by hazelcast.

the class TestSupportTest method test_processorSupplierHasJetInstance.

@Test
public void test_processorSupplierHasJetInstance() {
    HazelcastInstance hazelcastInstance = mockHazelcastInstance();
    boolean[] called = { false };
    verifyProcessor(new ProcessorSupplier() {

        @Override
        public void init(@Nonnull Context context) {
            assertSame(context.hazelcastInstance(), hazelcastInstance);
            called[0] = true;
        }

        @Nonnull
        @Override
        public Collection<? extends Processor> get(int count) {
            assertEquals(1, count);
            return singletonList(new MockP());
        }
    }).hazelcastInstance(hazelcastInstance).expectOutput(emptyList());
    assertTrue(called[0]);
}