Search in sources :

Example 16 with ProcessorMetaSupplier

use of com.hazelcast.jet.core.ProcessorMetaSupplier in project hazelcast by hazelcast.

the class ExecutionPlanBuilder method createExecutionPlans.

@SuppressWarnings("checkstyle:ParameterNumber")
public static Map<MemberInfo, ExecutionPlan> createExecutionPlans(NodeEngineImpl nodeEngine, List<MemberInfo> memberInfos, DAG dag, long jobId, long executionId, JobConfig jobConfig, long lastSnapshotId, boolean isLightJob, Subject subject) {
    final int defaultParallelism = nodeEngine.getConfig().getJetConfig().getCooperativeThreadCount();
    final Map<MemberInfo, int[]> partitionsByMember = getPartitionAssignment(nodeEngine, memberInfos);
    final Map<Address, int[]> partitionsByAddress = partitionsByMember.entrySet().stream().collect(toMap(en -> en.getKey().getAddress(), Entry::getValue));
    final List<Address> addresses = toList(partitionsByMember.keySet(), MemberInfo::getAddress);
    final int clusterSize = partitionsByMember.size();
    final boolean isJobDistributed = clusterSize > 1;
    final EdgeConfig defaultEdgeConfig = nodeEngine.getConfig().getJetConfig().getDefaultEdgeConfig();
    final Map<MemberInfo, ExecutionPlan> plans = new HashMap<>();
    int memberIndex = 0;
    for (MemberInfo member : partitionsByMember.keySet()) {
        plans.put(member, new ExecutionPlan(partitionsByAddress, jobConfig, lastSnapshotId, memberIndex++, clusterSize, isLightJob, subject));
    }
    final Map<String, Integer> vertexIdMap = assignVertexIds(dag);
    for (Entry<String, Integer> entry : vertexIdMap.entrySet()) {
        final Vertex vertex = dag.getVertex(entry.getKey());
        assert vertex != null;
        final ProcessorMetaSupplier metaSupplier = vertex.getMetaSupplier();
        final int vertexId = entry.getValue();
        // The local parallelism determination here is effective only
        // in jobs submitted as DAG. Otherwise, in jobs submitted as
        // pipeline, we are already doing this determination while
        // converting it to DAG and there is no vertex left with LP=-1.
        final int localParallelism = vertex.determineLocalParallelism(defaultParallelism);
        final int totalParallelism = localParallelism * clusterSize;
        final List<EdgeDef> inbound = toEdgeDefs(dag.getInboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getSourceName()), isJobDistributed);
        final List<EdgeDef> outbound = toEdgeDefs(dag.getOutboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getDestName()), isJobDistributed);
        String prefix = prefix(jobConfig.getName(), jobId, vertex.getName(), "#PMS");
        ILogger logger = prefixedLogger(nodeEngine.getLogger(metaSupplier.getClass()), prefix);
        JetServiceBackend jetBackend = nodeEngine.getService(JetServiceBackend.SERVICE_NAME);
        JobClassLoaderService jobClassLoaderService = jetBackend.getJobClassLoaderService();
        ClassLoader processorClassLoader = jobClassLoaderService.getClassLoader(jobId);
        try {
            doWithClassLoader(processorClassLoader, () -> metaSupplier.init(new MetaSupplierCtx(nodeEngine, jobId, executionId, jobConfig, logger, vertex.getName(), localParallelism, totalParallelism, clusterSize, isLightJob, partitionsByAddress, subject, processorClassLoader)));
        } catch (Exception e) {
            throw sneakyThrow(e);
        }
        Function<? super Address, ? extends ProcessorSupplier> procSupplierFn = doWithClassLoader(processorClassLoader, () -> metaSupplier.get(addresses));
        for (Entry<MemberInfo, ExecutionPlan> e : plans.entrySet()) {
            final ProcessorSupplier processorSupplier = doWithClassLoader(processorClassLoader, () -> procSupplierFn.apply(e.getKey().getAddress()));
            if (!isLightJob) {
                // We avoid the check for light jobs - the user will get the error anyway, but maybe with less information.
                // And we can recommend the user to use normal job to have more checks.
                checkSerializable(processorSupplier, "ProcessorSupplier in vertex '" + vertex.getName() + '\'');
            }
            final VertexDef vertexDef = new VertexDef(vertexId, vertex.getName(), processorSupplier, localParallelism);
            vertexDef.addInboundEdges(inbound);
            vertexDef.addOutboundEdges(outbound);
            e.getValue().addVertex(vertexDef);
        }
    }
    return plans;
}
Also used : IntStream(java.util.stream.IntStream) Address(com.hazelcast.cluster.Address) EdgeConfig(com.hazelcast.jet.config.EdgeConfig) Util.checkSerializable(com.hazelcast.jet.impl.util.Util.checkSerializable) Util.doWithClassLoader(com.hazelcast.jet.impl.util.Util.doWithClassLoader) HashMap(java.util.HashMap) ExceptionUtil.sneakyThrow(com.hazelcast.jet.impl.util.ExceptionUtil.sneakyThrow) Function(java.util.function.Function) ArrayList(java.util.ArrayList) PrefixedLogger.prefixedLogger(com.hazelcast.jet.impl.util.PrefixedLogger.prefixedLogger) LinkedHashMap(java.util.LinkedHashMap) Collectors.toMap(java.util.stream.Collectors.toMap) ILogger(com.hazelcast.logging.ILogger) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) Map(java.util.Map) DAG(com.hazelcast.jet.core.DAG) Edge(com.hazelcast.jet.core.Edge) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) Tuple2(com.hazelcast.jet.datamodel.Tuple2) NodeEngineImpl(com.hazelcast.spi.impl.NodeEngineImpl) PrefixedLogger.prefix(com.hazelcast.jet.impl.util.PrefixedLogger.prefix) NodeEngine(com.hazelcast.spi.impl.NodeEngine) Util.toList(com.hazelcast.jet.impl.util.Util.toList) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) IPartitionService(com.hazelcast.internal.partition.IPartitionService) JobConfig(com.hazelcast.jet.config.JobConfig) MetaSupplierCtx(com.hazelcast.jet.impl.execution.init.Contexts.MetaSupplierCtx) Collectors(java.util.stream.Collectors) Subject(javax.security.auth.Subject) Vertex(com.hazelcast.jet.core.Vertex) List(java.util.List) Tuple2.tuple2(com.hazelcast.jet.datamodel.Tuple2.tuple2) FunctionEx.identity(com.hazelcast.function.FunctionEx.identity) Entry(java.util.Map.Entry) JobClassLoaderService(com.hazelcast.jet.impl.JobClassLoaderService) JetServiceBackend(com.hazelcast.jet.impl.JetServiceBackend) Vertex(com.hazelcast.jet.core.Vertex) Address(com.hazelcast.cluster.Address) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) Util.doWithClassLoader(com.hazelcast.jet.impl.util.Util.doWithClassLoader) ILogger(com.hazelcast.logging.ILogger) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) JetServiceBackend(com.hazelcast.jet.impl.JetServiceBackend) EdgeConfig(com.hazelcast.jet.config.EdgeConfig) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) JobClassLoaderService(com.hazelcast.jet.impl.JobClassLoaderService) MetaSupplierCtx(com.hazelcast.jet.impl.execution.init.Contexts.MetaSupplierCtx)

Example 17 with ProcessorMetaSupplier

use of com.hazelcast.jet.core.ProcessorMetaSupplier in project hazelcast by hazelcast.

the class CdcSinks method sink.

@Nonnull
private static <K, V> Sink<ChangeRecord> sink(@Nonnull String name, @Nonnull String map, @Nullable ClientConfig clientConfig, @Nonnull FunctionEx<? super ChangeRecord, ? extends K> keyFn, @Nonnull FunctionEx<? super ChangeRecord, ? extends V> valueFn) {
    FunctionEx<? super ChangeRecord, ? extends V> toValueFn = record -> DELETE.equals(record.operation()) ? null : valueFn.apply(record);
    String clientXml = asXmlString(clientConfig);
    ProcessorSupplier supplier = AbstractHazelcastConnectorSupplier.ofMap(clientXml, procFn(name, map, clientXml, keyFn, toValueFn));
    ProcessorMetaSupplier metaSupplier = ProcessorMetaSupplier.of(mapUpdatePermission(clientXml, name), supplier);
    return new SinkImpl<>(name, metaSupplier, DISTRIBUTED_PARTITIONED, keyFn);
}
Also used : HazelcastInstance(com.hazelcast.core.HazelcastInstance) FunctionEx(com.hazelcast.function.FunctionEx) DELETE(com.hazelcast.jet.cdc.Operation.DELETE) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) Processor(com.hazelcast.jet.core.Processor) DISTRIBUTED_PARTITIONED(com.hazelcast.jet.impl.pipeline.SinkImpl.Type.DISTRIBUTED_PARTITIONED) Collections.singletonList(java.util.Collections.singletonList) PermissionsUtil.mapUpdatePermission(com.hazelcast.security.PermissionsUtil.mapUpdatePermission) List(java.util.List) Permission(java.security.Permission) AbstractHazelcastConnectorSupplier(com.hazelcast.jet.impl.connector.AbstractHazelcastConnectorSupplier) HazelcastProperty(com.hazelcast.spi.properties.HazelcastProperty) ClientConfig(com.hazelcast.client.config.ClientConfig) ImdgUtil.asXmlString(com.hazelcast.jet.impl.util.ImdgUtil.asXmlString) Nonnull(javax.annotation.Nonnull) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) SinkImpl(com.hazelcast.jet.impl.pipeline.SinkImpl) Sink(com.hazelcast.jet.pipeline.Sink) Nullable(javax.annotation.Nullable) SECONDS(java.util.concurrent.TimeUnit.SECONDS) WriteCdcP(com.hazelcast.jet.cdc.impl.WriteCdcP) IMap(com.hazelcast.map.IMap) SinkImpl(com.hazelcast.jet.impl.pipeline.SinkImpl) ImdgUtil.asXmlString(com.hazelcast.jet.impl.util.ImdgUtil.asXmlString) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) Nonnull(javax.annotation.Nonnull)

Example 18 with ProcessorMetaSupplier

use of com.hazelcast.jet.core.ProcessorMetaSupplier in project hazelcast by hazelcast.

the class TestSupportTest method test_processorMetaSupplierHasJetInstance.

@Test
public void test_processorMetaSupplierHasJetInstance() {
    HazelcastInstance hazelcastInstance = mockHazelcastInstance();
    boolean[] called = { false };
    verifyProcessor(new ProcessorMetaSupplier() {

        @Override
        public void init(@Nonnull Context context) {
            assertSame(context.hazelcastInstance(), hazelcastInstance);
            called[0] = true;
        }

        @Nonnull
        @Override
        public Function<? super Address, ? extends ProcessorSupplier> get(@Nonnull List<Address> addresses) {
            return a -> ProcessorSupplier.of(MockP::new);
        }
    }).hazelcastInstance(hazelcastInstance).expectOutput(emptyList());
    assertTrue(called[0]);
}
Also used : Function(java.util.function.Function) HazelcastInstance(com.hazelcast.core.HazelcastInstance) Address(com.hazelcast.cluster.Address) Nonnull(javax.annotation.Nonnull) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) QuickTest(com.hazelcast.test.annotation.QuickTest) Test(org.junit.Test)

Example 19 with ProcessorMetaSupplier

use of com.hazelcast.jet.core.ProcessorMetaSupplier in project hazelcast by hazelcast.

the class WriteFilePTest method test_abortUnfinishedTransaction_whenNoItemsProcessed.

@Test
public void test_abortUnfinishedTransaction_whenNoItemsProcessed() throws Exception {
    // test for https://github.com/hazelcast/hazelcast/issues/19774
    ProcessorMetaSupplier metaSupplier = writeFileP(directory.toString(), StandardCharsets.UTF_8, null, DISABLE_ROLLING, true, Objects::toString);
    TestProcessorContext processorContext = new TestProcessorContext().setProcessingGuarantee(EXACTLY_ONCE);
    @SuppressWarnings("unchecked") WriteFileP<Integer> processor = (WriteFileP<Integer>) TestSupport.supplierFrom(metaSupplier).get();
    processor.init(new TestOutbox(new int[] { 128 }, 128), processorContext);
    processor.process(0, new TestInbox(singletonList(42)));
    assertTrue(processor.snapshotCommitPrepare());
    checkFileContents(0, 0, true, true, true);
    // Now a tmp file is created. Let's simulate that the prepared snapshot wasn't successful and
    // the job restarted
    @SuppressWarnings("unchecked") WriteFileP<Integer> processor2 = (WriteFileP<Integer>) TestSupport.supplierFrom(metaSupplier).get();
    processor2.init(new TestOutbox(128), processorContext);
    processor2.close();
    // now there should be no temp files
    checkFileContents(0, 0, true, false, true);
}
Also used : TestInbox(com.hazelcast.jet.core.test.TestInbox) TestOutbox(com.hazelcast.jet.core.test.TestOutbox) Objects(java.util.Objects) TestProcessorContext(com.hazelcast.jet.core.test.TestProcessorContext) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) QuickTest(com.hazelcast.test.annotation.QuickTest) Test(org.junit.Test)

Example 20 with ProcessorMetaSupplier

use of com.hazelcast.jet.core.ProcessorMetaSupplier in project hazelcast-jet by hazelcast.

the class ExecutionPlanBuilder method createExecutionPlans.

public static Map<MemberInfo, ExecutionPlan> createExecutionPlans(NodeEngine nodeEngine, MembersView membersView, DAG dag, JobConfig jobConfig, long lastSnapshotId) {
    final JetInstance instance = getJetInstance(nodeEngine);
    final int defaultParallelism = instance.getConfig().getInstanceConfig().getCooperativeThreadCount();
    final Collection<MemberInfo> members = new HashSet<>(membersView.size());
    final Address[] partitionOwners = new Address[nodeEngine.getPartitionService().getPartitionCount()];
    initPartitionOwnersAndMembers(nodeEngine, membersView, members, partitionOwners);
    final List<Address> addresses = members.stream().map(MemberInfo::getAddress).collect(toList());
    final int clusterSize = members.size();
    final boolean isJobDistributed = clusterSize > 1;
    final EdgeConfig defaultEdgeConfig = instance.getConfig().getDefaultEdgeConfig();
    final Map<MemberInfo, ExecutionPlan> plans = members.stream().collect(toMap(m -> m, m -> new ExecutionPlan(partitionOwners, jobConfig, lastSnapshotId)));
    final Map<String, Integer> vertexIdMap = assignVertexIds(dag);
    for (Entry<String, Integer> entry : vertexIdMap.entrySet()) {
        final Vertex vertex = dag.getVertex(entry.getKey());
        final ProcessorMetaSupplier metaSupplier = vertex.getMetaSupplier();
        final int vertexId = entry.getValue();
        final int localParallelism = determineParallelism(vertex, metaSupplier.preferredLocalParallelism(), defaultParallelism);
        final int totalParallelism = localParallelism * clusterSize;
        final List<EdgeDef> inbound = toEdgeDefs(dag.getInboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getSourceName()), isJobDistributed);
        final List<EdgeDef> outbound = toEdgeDefs(dag.getOutboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getDestName()), isJobDistributed);
        final ILogger logger = nodeEngine.getLogger(String.format("%s.%s#ProcessorMetaSupplier", metaSupplier.getClass().getName(), vertex.getName()));
        metaSupplier.init(new MetaSupplierCtx(instance, logger, vertex.getName(), localParallelism, totalParallelism));
        Function<Address, ProcessorSupplier> procSupplierFn = metaSupplier.get(addresses);
        int procIdxOffset = 0;
        for (Entry<MemberInfo, ExecutionPlan> e : plans.entrySet()) {
            final ProcessorSupplier processorSupplier = procSupplierFn.apply(e.getKey().getAddress());
            checkSerializable(processorSupplier, "ProcessorSupplier in vertex '" + vertex.getName() + '\'');
            final VertexDef vertexDef = new VertexDef(vertexId, vertex.getName(), processorSupplier, procIdxOffset, localParallelism, totalParallelism);
            vertexDef.addInboundEdges(inbound);
            vertexDef.addOutboundEdges(outbound);
            e.getValue().addVertex(vertexDef);
            procIdxOffset += localParallelism;
        }
    }
    return plans;
}
Also used : EdgeConfig(com.hazelcast.jet.config.EdgeConfig) JetInstance(com.hazelcast.jet.JetInstance) Util.checkSerializable(com.hazelcast.jet.impl.util.Util.checkSerializable) Address(com.hazelcast.nio.Address) LOCAL_PARALLELISM_USE_DEFAULT(com.hazelcast.jet.core.Vertex.LOCAL_PARALLELISM_USE_DEFAULT) Function(java.util.function.Function) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) Collectors.toMap(java.util.stream.Collectors.toMap) ILogger(com.hazelcast.logging.ILogger) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) Map(java.util.Map) MembersView(com.hazelcast.internal.cluster.impl.MembersView) DAG(com.hazelcast.jet.core.DAG) Edge(com.hazelcast.jet.core.Edge) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) Integer.min(java.lang.Integer.min) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) Collection(java.util.Collection) Util.getJetInstance(com.hazelcast.jet.impl.util.Util.getJetInstance) JobConfig(com.hazelcast.jet.config.JobConfig) IPartitionService(com.hazelcast.spi.partition.IPartitionService) MetaSupplierCtx(com.hazelcast.jet.impl.execution.init.Contexts.MetaSupplierCtx) NodeEngine(com.hazelcast.spi.NodeEngine) Vertex(com.hazelcast.jet.core.Vertex) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) Entry(java.util.Map.Entry) TopologyChangedException(com.hazelcast.jet.core.TopologyChangedException) Vertex(com.hazelcast.jet.core.Vertex) Address(com.hazelcast.nio.Address) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) ILogger(com.hazelcast.logging.ILogger) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) HashSet(java.util.HashSet) JetInstance(com.hazelcast.jet.JetInstance) Util.getJetInstance(com.hazelcast.jet.impl.util.Util.getJetInstance) EdgeConfig(com.hazelcast.jet.config.EdgeConfig) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) MetaSupplierCtx(com.hazelcast.jet.impl.execution.init.Contexts.MetaSupplierCtx)

Aggregations

ProcessorMetaSupplier (com.hazelcast.jet.core.ProcessorMetaSupplier)21 ProcessorSupplier (com.hazelcast.jet.core.ProcessorSupplier)11 Nonnull (javax.annotation.Nonnull)8 Test (org.junit.Test)6 Vertex (com.hazelcast.jet.core.Vertex)5 QuickTest (com.hazelcast.test.annotation.QuickTest)5 List (java.util.List)5 Address (com.hazelcast.cluster.Address)4 FunctionEx (com.hazelcast.function.FunctionEx)4 DAG (com.hazelcast.jet.core.DAG)4 TestProcessorContext (com.hazelcast.jet.core.test.TestProcessorContext)4 JetSqlRow (com.hazelcast.sql.impl.row.JetSqlRow)4 ParallelJVMTest (com.hazelcast.test.annotation.ParallelJVMTest)4 Map (java.util.Map)4 EventTimePolicy (com.hazelcast.jet.core.EventTimePolicy)3 Function (java.util.function.Function)3 Nullable (javax.annotation.Nullable)3 HazelcastInstance (com.hazelcast.core.HazelcastInstance)2 MemberInfo (com.hazelcast.internal.cluster.MemberInfo)2 EdgeConfig (com.hazelcast.jet.config.EdgeConfig)2