Search in sources :

Example 1 with EXECUTION

use of com.hazelcast.jet.impl.JobClassLoaderService.JobPhase.EXECUTION in project hazelcast by hazelcast.

the class JobExecutionService method checkExecutions.

/**
 * See also javadoc at {@link CheckLightJobsOperation}.
 */
private void checkExecutions() {
    try {
        long now = System.nanoTime();
        long uninitializedContextThreshold = now - UNINITIALIZED_CONTEXT_MAX_AGE_NS;
        Map<Address, List<Long>> executionsPerMember = new HashMap<>();
        for (ExecutionContext ctx : executionContexts.values()) {
            if (!ctx.isLightJob()) {
                continue;
            }
            Address coordinator = ctx.coordinator();
            if (coordinator != null) {
                // if coordinator is known, add execution to the list to check
                executionsPerMember.computeIfAbsent(coordinator, k -> new ArrayList<>()).add(ctx.executionId());
            } else {
                // if coordinator is not known, remove execution if it's not known for too long
                if (ctx.getCreatedOn() <= uninitializedContextThreshold) {
                    LoggingUtil.logFine(logger, "Terminating light job %s because it wasn't initialized during %d seconds", idToString(ctx.executionId()), NANOSECONDS.toSeconds(UNINITIALIZED_CONTEXT_MAX_AGE_NS));
                    terminateExecution0(ctx, TerminationMode.CANCEL_FORCEFUL, new CancellationException());
                }
            }
        }
        // submit the query to the coordinator
        for (Entry<Address, List<Long>> en : executionsPerMember.entrySet()) {
            long[] executionIds = en.getValue().stream().mapToLong(Long::longValue).toArray();
            Operation op = new CheckLightJobsOperation(executionIds);
            InvocationFuture<long[]> future = nodeEngine.getOperationService().createInvocationBuilder(JetServiceBackend.SERVICE_NAME, op, en.getKey()).invoke();
            future.whenComplete((r, t) -> {
                if (t instanceof TargetNotMemberException) {
                    // if the target isn't a member, then all executions are unknown
                    r = executionIds;
                } else if (t != null) {
                    logger.warning("Failed to check light job state with coordinator " + en.getKey() + ": " + t, t);
                    return;
                }
                assert r != null;
                for (long executionId : r) {
                    ExecutionContext execCtx = executionContexts.get(executionId);
                    if (execCtx != null) {
                        logger.fine("Terminating light job " + idToString(executionId) + " because the coordinator doesn't know it");
                        terminateExecution0(execCtx, TerminationMode.CANCEL_FORCEFUL, new CancellationException());
                    }
                }
            });
        }
        // clean up failedJobs
        failedJobs.values().removeIf(expiryTime -> expiryTime < now);
    } catch (Throwable e) {
        logger.severe("Failed to query live light executions: " + e, e);
    }
}
Also used : Address(com.hazelcast.cluster.Address) HazelcastInstanceNotActiveException(com.hazelcast.core.HazelcastInstanceNotActiveException) InvocationFuture(com.hazelcast.spi.impl.operationservice.impl.InvocationFuture) ScheduledFuture(java.util.concurrent.ScheduledFuture) Member(com.hazelcast.cluster.Member) UnaryOperator(java.util.function.UnaryOperator) JobTerminateRequestedException(com.hazelcast.jet.impl.exception.JobTerminateRequestedException) JetDelegatingClassLoader(com.hazelcast.jet.impl.deployment.JetDelegatingClassLoader) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) MwCounter(com.hazelcast.internal.util.counters.MwCounter) Map(java.util.Map) DynamicMetricsProvider(com.hazelcast.internal.metrics.DynamicMetricsProvider) Counter(com.hazelcast.internal.util.counters.Counter) Collectors.toSet(java.util.stream.Collectors.toSet) ExecutionContext(com.hazelcast.jet.impl.execution.ExecutionContext) RetryableHazelcastException(com.hazelcast.spi.exception.RetryableHazelcastException) CancellationException(java.util.concurrent.CancellationException) Probe(com.hazelcast.internal.metrics.Probe) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) TargetNotMemberException(com.hazelcast.spi.exception.TargetNotMemberException) Objects(java.util.Objects) SenderReceiverKey(com.hazelcast.jet.impl.execution.ExecutionContext.SenderReceiverKey) List(java.util.List) Util.idToString(com.hazelcast.jet.Util.idToString) ExecutionPlan(com.hazelcast.jet.impl.execution.init.ExecutionPlan) MetricNames(com.hazelcast.jet.core.metrics.MetricNames) Entry(java.util.Map.Entry) TopologyChangedException(com.hazelcast.jet.core.TopologyChangedException) LoggingUtil(com.hazelcast.jet.impl.util.LoggingUtil) MetricsRegistry(com.hazelcast.internal.metrics.MetricsRegistry) NANOSECONDS(java.util.concurrent.TimeUnit.NANOSECONDS) MembershipManager(com.hazelcast.internal.cluster.impl.MembershipManager) MetricsCompressor(com.hazelcast.internal.metrics.impl.MetricsCompressor) Util.doWithClassLoader(com.hazelcast.jet.impl.util.Util.doWithClassLoader) SenderTasklet(com.hazelcast.jet.impl.execution.SenderTasklet) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) MINUTES(java.util.concurrent.TimeUnit.MINUTES) Function(java.util.function.Function) ExecutionNotFoundException(com.hazelcast.jet.impl.exception.ExecutionNotFoundException) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) EXECUTION(com.hazelcast.jet.impl.JobClassLoaderService.JobPhase.EXECUTION) MetricDescriptor(com.hazelcast.internal.metrics.MetricDescriptor) Collections.newSetFromMap(java.util.Collections.newSetFromMap) ILogger(com.hazelcast.logging.ILogger) Operation(com.hazelcast.spi.impl.operationservice.Operation) ExceptionUtil.withTryCatch(com.hazelcast.jet.impl.util.ExceptionUtil.withTryCatch) TaskletExecutionService(com.hazelcast.jet.impl.execution.TaskletExecutionService) ClusterServiceImpl(com.hazelcast.internal.cluster.impl.ClusterServiceImpl) Nonnull(javax.annotation.Nonnull) CheckLightJobsOperation(com.hazelcast.jet.impl.operation.CheckLightJobsOperation) Nullable(javax.annotation.Nullable) NodeEngineImpl(com.hazelcast.spi.impl.NodeEngineImpl) MemberLeftException(com.hazelcast.core.MemberLeftException) MetricsCollectionContext(com.hazelcast.internal.metrics.MetricsCollectionContext) MetricsCollector(com.hazelcast.internal.metrics.collectors.MetricsCollector) RawJobMetrics(com.hazelcast.jet.impl.metrics.RawJobMetrics) MetricTags(com.hazelcast.jet.core.metrics.MetricTags) TriggerMemberListPublishOp(com.hazelcast.internal.cluster.impl.operations.TriggerMemberListPublishOp) ExceptionUtil.peel(com.hazelcast.jet.impl.util.ExceptionUtil.peel) Util.jobIdAndExecutionId(com.hazelcast.jet.impl.util.Util.jobIdAndExecutionId) Util(com.hazelcast.jet.Util) SECONDS(java.util.concurrent.TimeUnit.SECONDS) CheckLightJobsOperation(com.hazelcast.jet.impl.operation.CheckLightJobsOperation) Address(com.hazelcast.cluster.Address) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Operation(com.hazelcast.spi.impl.operationservice.Operation) CheckLightJobsOperation(com.hazelcast.jet.impl.operation.CheckLightJobsOperation) TargetNotMemberException(com.hazelcast.spi.exception.TargetNotMemberException) ExecutionContext(com.hazelcast.jet.impl.execution.ExecutionContext) CancellationException(java.util.concurrent.CancellationException) List(java.util.List) ArrayList(java.util.ArrayList)

Example 2 with EXECUTION

use of com.hazelcast.jet.impl.JobClassLoaderService.JobPhase.EXECUTION in project hazelcast by hazelcast.

the class JobExecutionService method runLightJob.

public CompletableFuture<RawJobMetrics> runLightJob(long jobId, long executionId, Address coordinator, int coordinatorMemberListVersion, Set<MemberInfo> participants, ExecutionPlan plan) {
    assert executionId == jobId : "executionId(" + idToString(executionId) + ") != jobId(" + idToString(jobId) + ")";
    verifyClusterInformation(jobId, executionId, coordinator, coordinatorMemberListVersion, participants);
    failIfNotRunning();
    ExecutionContext execCtx;
    synchronized (mutex) {
        addExecutionContextJobId(jobId, executionId, coordinator);
        execCtx = executionContexts.computeIfAbsent(executionId, x -> new ExecutionContext(nodeEngine, jobId, executionId, true));
    }
    try {
        Set<Address> addresses = participants.stream().map(MemberInfo::getAddress).collect(toSet());
        ClassLoader jobCl = jobClassloaderService.getClassLoader(jobId);
        // We don't create the CL for light jobs.
        assert jobClassloaderService.getClassLoader(jobId) == null;
        doWithClassLoader(jobCl, () -> execCtx.initialize(coordinator, addresses, plan));
    } catch (Throwable e) {
        completeExecution(execCtx, new CancellationException());
        throw e;
    }
    // initial log entry with all of jobId, jobName, executionId
    if (logger.isFineEnabled()) {
        logger.fine("Execution plan for light job ID=" + idToString(jobId) + ", jobName=" + (execCtx.jobName() != null ? '\'' + execCtx.jobName() + '\'' : "null") + ", executionId=" + idToString(executionId) + " initialized, will start the execution");
    }
    return beginExecution0(execCtx, false);
}
Also used : Address(com.hazelcast.cluster.Address) HazelcastInstanceNotActiveException(com.hazelcast.core.HazelcastInstanceNotActiveException) InvocationFuture(com.hazelcast.spi.impl.operationservice.impl.InvocationFuture) ScheduledFuture(java.util.concurrent.ScheduledFuture) Member(com.hazelcast.cluster.Member) UnaryOperator(java.util.function.UnaryOperator) JobTerminateRequestedException(com.hazelcast.jet.impl.exception.JobTerminateRequestedException) JetDelegatingClassLoader(com.hazelcast.jet.impl.deployment.JetDelegatingClassLoader) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) MwCounter(com.hazelcast.internal.util.counters.MwCounter) Map(java.util.Map) DynamicMetricsProvider(com.hazelcast.internal.metrics.DynamicMetricsProvider) Counter(com.hazelcast.internal.util.counters.Counter) Collectors.toSet(java.util.stream.Collectors.toSet) ExecutionContext(com.hazelcast.jet.impl.execution.ExecutionContext) RetryableHazelcastException(com.hazelcast.spi.exception.RetryableHazelcastException) CancellationException(java.util.concurrent.CancellationException) Probe(com.hazelcast.internal.metrics.Probe) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) TargetNotMemberException(com.hazelcast.spi.exception.TargetNotMemberException) Objects(java.util.Objects) SenderReceiverKey(com.hazelcast.jet.impl.execution.ExecutionContext.SenderReceiverKey) List(java.util.List) Util.idToString(com.hazelcast.jet.Util.idToString) ExecutionPlan(com.hazelcast.jet.impl.execution.init.ExecutionPlan) MetricNames(com.hazelcast.jet.core.metrics.MetricNames) Entry(java.util.Map.Entry) TopologyChangedException(com.hazelcast.jet.core.TopologyChangedException) LoggingUtil(com.hazelcast.jet.impl.util.LoggingUtil) MetricsRegistry(com.hazelcast.internal.metrics.MetricsRegistry) NANOSECONDS(java.util.concurrent.TimeUnit.NANOSECONDS) MembershipManager(com.hazelcast.internal.cluster.impl.MembershipManager) MetricsCompressor(com.hazelcast.internal.metrics.impl.MetricsCompressor) Util.doWithClassLoader(com.hazelcast.jet.impl.util.Util.doWithClassLoader) SenderTasklet(com.hazelcast.jet.impl.execution.SenderTasklet) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) MINUTES(java.util.concurrent.TimeUnit.MINUTES) Function(java.util.function.Function) ExecutionNotFoundException(com.hazelcast.jet.impl.exception.ExecutionNotFoundException) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) EXECUTION(com.hazelcast.jet.impl.JobClassLoaderService.JobPhase.EXECUTION) MetricDescriptor(com.hazelcast.internal.metrics.MetricDescriptor) Collections.newSetFromMap(java.util.Collections.newSetFromMap) ILogger(com.hazelcast.logging.ILogger) Operation(com.hazelcast.spi.impl.operationservice.Operation) ExceptionUtil.withTryCatch(com.hazelcast.jet.impl.util.ExceptionUtil.withTryCatch) TaskletExecutionService(com.hazelcast.jet.impl.execution.TaskletExecutionService) ClusterServiceImpl(com.hazelcast.internal.cluster.impl.ClusterServiceImpl) Nonnull(javax.annotation.Nonnull) CheckLightJobsOperation(com.hazelcast.jet.impl.operation.CheckLightJobsOperation) Nullable(javax.annotation.Nullable) NodeEngineImpl(com.hazelcast.spi.impl.NodeEngineImpl) MemberLeftException(com.hazelcast.core.MemberLeftException) MetricsCollectionContext(com.hazelcast.internal.metrics.MetricsCollectionContext) MetricsCollector(com.hazelcast.internal.metrics.collectors.MetricsCollector) RawJobMetrics(com.hazelcast.jet.impl.metrics.RawJobMetrics) MetricTags(com.hazelcast.jet.core.metrics.MetricTags) TriggerMemberListPublishOp(com.hazelcast.internal.cluster.impl.operations.TriggerMemberListPublishOp) ExceptionUtil.peel(com.hazelcast.jet.impl.util.ExceptionUtil.peel) Util.jobIdAndExecutionId(com.hazelcast.jet.impl.util.Util.jobIdAndExecutionId) Util(com.hazelcast.jet.Util) SECONDS(java.util.concurrent.TimeUnit.SECONDS) ExecutionContext(com.hazelcast.jet.impl.execution.ExecutionContext) Address(com.hazelcast.cluster.Address) CancellationException(java.util.concurrent.CancellationException) JetDelegatingClassLoader(com.hazelcast.jet.impl.deployment.JetDelegatingClassLoader) Util.doWithClassLoader(com.hazelcast.jet.impl.util.Util.doWithClassLoader)

Aggregations

Address (com.hazelcast.cluster.Address)2 Member (com.hazelcast.cluster.Member)2 HazelcastInstanceNotActiveException (com.hazelcast.core.HazelcastInstanceNotActiveException)2 MemberLeftException (com.hazelcast.core.MemberLeftException)2 MemberInfo (com.hazelcast.internal.cluster.MemberInfo)2 ClusterServiceImpl (com.hazelcast.internal.cluster.impl.ClusterServiceImpl)2 MembershipManager (com.hazelcast.internal.cluster.impl.MembershipManager)2 TriggerMemberListPublishOp (com.hazelcast.internal.cluster.impl.operations.TriggerMemberListPublishOp)2 DynamicMetricsProvider (com.hazelcast.internal.metrics.DynamicMetricsProvider)2 MetricDescriptor (com.hazelcast.internal.metrics.MetricDescriptor)2 MetricsCollectionContext (com.hazelcast.internal.metrics.MetricsCollectionContext)2 MetricsRegistry (com.hazelcast.internal.metrics.MetricsRegistry)2 Probe (com.hazelcast.internal.metrics.Probe)2 MetricsCollector (com.hazelcast.internal.metrics.collectors.MetricsCollector)2 MetricsCompressor (com.hazelcast.internal.metrics.impl.MetricsCompressor)2 Counter (com.hazelcast.internal.util.counters.Counter)2 MwCounter (com.hazelcast.internal.util.counters.MwCounter)2 Util (com.hazelcast.jet.Util)2 Util.idToString (com.hazelcast.jet.Util.idToString)2 TopologyChangedException (com.hazelcast.jet.core.TopologyChangedException)2