Examples with JobClassLoaderService com.hazelcast.jet.impl.JobClassLoaderService used on opensource projects

Example 1 with JobClassLoaderService

use of com.hazelcast.jet.impl.JobClassLoaderService in project hazelcast by hazelcast.

the class JetTestSupport method shutdownJobsAndGetLeakedClassLoaders.

@Nonnull
private Map<Long, String> shutdownJobsAndGetLeakedClassLoaders() {
    Map<Long, String> leakedClassloaders = new HashMap<>();
    Collection<HazelcastInstance> instances = instanceFactory.getAllHazelcastInstances();
    for (HazelcastInstance instance : instances) {
        if (instance.getConfig().getJetConfig().isEnabled()) {
            // Some tests leave jobs running, which keeps job classloader, shut down all running/starting jobs
            JetService jet = instance.getJet();
            List<Job> jobs = jet.getJobs();
            for (Job job : jobs) {
                ditchJob(job, instances.toArray(new HazelcastInstance[instances.size()]));
            }
            JobClassLoaderService jobClassLoaderService = ((HazelcastInstanceImpl) instance).node.getNodeEngine().<JetServiceBackend>getService(SERVICE_NAME).getJobClassLoaderService();
            Map<Long, ?> classLoaders = jobClassLoaderService.getClassLoaders();
            // The classloader cleanup is done asynchronously in some cases, wait up to 10s
            for (int i = 0; i < 100 && !classLoaders.isEmpty(); i++) {
                sleepMillis(100);
            }
            for (Entry<Long, ?> entry : classLoaders.entrySet()) {
                leakedClassloaders.put(entry.getKey(), entry.toString());
            }
        }
    }
    return leakedClassloaders;
}

Example 2 with JobClassLoaderService

use of com.hazelcast.jet.impl.JobClassLoaderService in project hazelcast by hazelcast.

the class ExecutionPlanBuilder method createExecutionPlans.

@SuppressWarnings("checkstyle:ParameterNumber")
public static Map<MemberInfo, ExecutionPlan> createExecutionPlans(NodeEngineImpl nodeEngine, List<MemberInfo> memberInfos, DAG dag, long jobId, long executionId, JobConfig jobConfig, long lastSnapshotId, boolean isLightJob, Subject subject) {
    final int defaultParallelism = nodeEngine.getConfig().getJetConfig().getCooperativeThreadCount();
    final Map<MemberInfo, int[]> partitionsByMember = getPartitionAssignment(nodeEngine, memberInfos);
    final Map<Address, int[]> partitionsByAddress = partitionsByMember.entrySet().stream().collect(toMap(en -> en.getKey().getAddress(), Entry::getValue));
    final List<Address> addresses = toList(partitionsByMember.keySet(), MemberInfo::getAddress);
    final int clusterSize = partitionsByMember.size();
    final boolean isJobDistributed = clusterSize > 1;
    final EdgeConfig defaultEdgeConfig = nodeEngine.getConfig().getJetConfig().getDefaultEdgeConfig();
    final Map<MemberInfo, ExecutionPlan> plans = new HashMap<>();
    int memberIndex = 0;
    for (MemberInfo member : partitionsByMember.keySet()) {
        plans.put(member, new ExecutionPlan(partitionsByAddress, jobConfig, lastSnapshotId, memberIndex++, clusterSize, isLightJob, subject));
    }
    final Map<String, Integer> vertexIdMap = assignVertexIds(dag);
    for (Entry<String, Integer> entry : vertexIdMap.entrySet()) {
        final Vertex vertex = dag.getVertex(entry.getKey());
        assert vertex != null;
        final ProcessorMetaSupplier metaSupplier = vertex.getMetaSupplier();
        final int vertexId = entry.getValue();
        // The local parallelism determination here is effective only
        // in jobs submitted as DAG. Otherwise, in jobs submitted as
        // pipeline, we are already doing this determination while
        // converting it to DAG and there is no vertex left with LP=-1.
        final int localParallelism = vertex.determineLocalParallelism(defaultParallelism);
        final int totalParallelism = localParallelism * clusterSize;
        final List<EdgeDef> inbound = toEdgeDefs(dag.getInboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getSourceName()), isJobDistributed);
        final List<EdgeDef> outbound = toEdgeDefs(dag.getOutboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getDestName()), isJobDistributed);
        String prefix = prefix(jobConfig.getName(), jobId, vertex.getName(), "#PMS");
        ILogger logger = prefixedLogger(nodeEngine.getLogger(metaSupplier.getClass()), prefix);
        JetServiceBackend jetBackend = nodeEngine.getService(JetServiceBackend.SERVICE_NAME);
        JobClassLoaderService jobClassLoaderService = jetBackend.getJobClassLoaderService();
        ClassLoader processorClassLoader = jobClassLoaderService.getClassLoader(jobId);
        try {
            doWithClassLoader(processorClassLoader, () -> metaSupplier.init(new MetaSupplierCtx(nodeEngine, jobId, executionId, jobConfig, logger, vertex.getName(), localParallelism, totalParallelism, clusterSize, isLightJob, partitionsByAddress, subject, processorClassLoader)));
        } catch (Exception e) {
            throw sneakyThrow(e);
        }
        Function<? super Address, ? extends ProcessorSupplier> procSupplierFn = doWithClassLoader(processorClassLoader, () -> metaSupplier.get(addresses));
        for (Entry<MemberInfo, ExecutionPlan> e : plans.entrySet()) {
            final ProcessorSupplier processorSupplier = doWithClassLoader(processorClassLoader, () -> procSupplierFn.apply(e.getKey().getAddress()));
            if (!isLightJob) {
                // We avoid the check for light jobs - the user will get the error anyway, but maybe with less information.
                // And we can recommend the user to use normal job to have more checks.
                checkSerializable(processorSupplier, "ProcessorSupplier in vertex '" + vertex.getName() + '\'');
            }
            final VertexDef vertexDef = new VertexDef(vertexId, vertex.getName(), processorSupplier, localParallelism);
            vertexDef.addInboundEdges(inbound);
            vertexDef.addOutboundEdges(outbound);
            e.getValue().addVertex(vertexDef);
        }
    }
    return plans;
}

Example 3 with JobClassLoaderService

use of com.hazelcast.jet.impl.JobClassLoaderService in project hazelcast by hazelcast.

the class ExecutionContext method completeExecution.

/**
 * Complete local execution. If local execution was started, it should be
 * called after execution has completed.
 */
public void completeExecution(Throwable error) {
    assert executionFuture == null || executionFuture.isDone() : "If execution was begun, then completeExecution() should not be called before execution is done.";
    if (!executionCompleted.compareAndSet(false, true)) {
        return;
    }
    for (Tasklet tasklet : tasklets) {
        try {
            tasklet.close();
        } catch (Throwable e) {
            logger.severe(jobNameAndExecutionId() + " encountered an exception in Processor.close(), ignoring it", e);
        }
    }
    JobClassLoaderService jobClassloaderService = jetServiceBackend.getJobClassLoaderService();
    ClassLoader classLoader = jobClassloaderService.getClassLoader(jobId);
    doWithClassLoader(classLoader, () -> {
        for (VertexDef vertex : vertices) {
            try {
                ClassLoader processorCl = isLightJob ? null : jobClassloaderService.getProcessorClassLoader(jobId, vertex.name());
                doWithClassLoader(processorCl, () -> vertex.processorSupplier().close(error));
            } catch (Throwable e) {
                logger.severe(jobNameAndExecutionId() + " encountered an exception in ProcessorSupplier.close(), ignoring it", e);
            }
        }
    });
    tempDirectories.forEach((k, dir) -> {
        try {
            IOUtil.delete(dir);
        } catch (Exception e) {
            logger.warning("Failed to delete temporary directory " + dir);
        }
    });
    if (serializationService != null) {
        serializationService.dispose();
    }
}

Example 4 with JobClassLoaderService

use of com.hazelcast.jet.impl.JobClassLoaderService in project hazelcast by hazelcast.

the class InitExecutionOperation method deserializePlan.

private ExecutionPlan deserializePlan(Data planBlob) {
    if (isLightJob) {
        return getNodeEngine().getSerializationService().toObject(planBlob);
    } else {
        JetServiceBackend service = getJetServiceBackend();
        JobConfig jobConfig = service.getJobConfig(jobId(), isLightJob);
        JobClassLoaderService jobClassloaderService = service.getJobClassLoaderService();
        ClassLoader cl = jobClassloaderService.getOrCreateClassLoader(jobConfig, jobId(), EXECUTION);
        try {
            jobClassloaderService.prepareProcessorClassLoaders(jobId());
            return deserializeWithCustomClassLoader(getNodeEngine().getSerializationService(), cl, planBlob);
        } finally {
            jobClassloaderService.clearProcessorClassLoaders();
        }
    }
}

Example 5 with JobClassLoaderService

use of com.hazelcast.jet.impl.JobClassLoaderService in project hazelcast by hazelcast.

the class ProcessorClassLoaderCleanupTest method processorClassLoaderRemovedAfterJobFinished.

@Test
public void processorClassLoaderRemovedAfterJobFinished() throws Exception {
    Pipeline p = Pipeline.create();
    StreamSource<SimpleEvent> source = TestSources.itemStream(1);
    p.readFrom(source).withoutTimestamps().setLocalParallelism(1).writeTo(Sinks.logger());
    JobConfig jobConfig = new JobConfig();
    jobConfig.addCustomClasspath(source.name(), jarFile.getName());
    Job job = jet.newJob(p, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);
    JetServiceBackend jetServiceBackend = ((HazelcastInstanceProxy) member).getOriginal().node.getNodeEngine().getService(JetServiceBackend.SERVICE_NAME);
    JobClassLoaderService jobClassLoaderService = jetServiceBackend.getJobClassLoaderService();
    ChildFirstClassLoader classLoader = (ChildFirstClassLoader) jobClassLoaderService.getProcessorClassLoader(job.getId(), source.name());
    job.suspend();
    assertJobStatusEventually(job, JobStatus.SUSPENDED);
    assertThat(classLoader.isClosed()).describedAs("classloader hasn't been closed").isTrue();
    assertThatThrownBy(() -> jobClassLoaderService.getProcessorClassLoader(job.getId(), source.name())).isInstanceOf(HazelcastException.class).hasMessageContaining("JobClassLoaders for jobId=" + Util.idToString(job.getId()) + " requested, but it does not exists");
}