Search in sources :

Example 1 with TerminatedWithSnapshotException

use of com.hazelcast.jet.impl.exception.TerminatedWithSnapshotException in project hazelcast by hazelcast.

the class MasterJobContext method getErrorFromResponses.

/**
 * <ul>
 * <li>Returns {@code null} if there is no failure
 * <li>Returns a {@link CancellationException} if the job is cancelled
 *     forcefully.
 * <li>Returns a {@link JobTerminateRequestedException} if the current
 *     execution is stopped due to a requested termination, except for
 *     CANCEL_GRACEFUL, in which case CancellationException is returned.
 * <li>If there is at least one user failure, such as an exception in user
 *     code (restartable or not), then returns that failure.
 * <li>Otherwise, the failure is because a job participant has left the
 *     cluster. In that case, it returns {@code TopologyChangeException} so
 *     that the job will be restarted
 * </ul>
 */
private Throwable getErrorFromResponses(String opName, Collection<Map.Entry<MemberInfo, Object>> responses) {
    if (isCancelled()) {
        logger.fine(mc.jobIdString() + " to be cancelled after " + opName);
        return new CancellationException();
    }
    Map<Boolean, List<Entry<Address, Object>>> grouped = responses.stream().map(en -> entry(en.getKey().getAddress(), en.getValue())).collect(partitioningBy(e1 -> e1.getValue() instanceof Throwable));
    int successfulMembersCount = grouped.getOrDefault(false, emptyList()).size();
    if (successfulMembersCount == mc.executionPlanMap().size()) {
        logger.fine(opName + " of " + mc.jobIdString() + " was successful");
        return null;
    }
    List<Entry<Address, Object>> failures = grouped.getOrDefault(true, emptyList());
    if (!failures.isEmpty()) {
        logger.fine(opName + " of " + mc.jobIdString() + " has failures: " + failures);
    }
    // other exceptions, ignore this and handle the other exception.
    if (failures.stream().allMatch(entry -> entry.getValue() instanceof TerminatedWithSnapshotException)) {
        assert opName.equals("Execution") : "opName is '" + opName + "', expected 'Execution'";
        logger.fine(opName + " of " + mc.jobIdString() + " terminated after a terminal snapshot");
        TerminationMode mode = requestedTerminationMode;
        assert mode != null && mode.isWithTerminalSnapshot() : "mode=" + mode;
        return mode == CANCEL_GRACEFUL ? new CancellationException() : new JobTerminateRequestedException(mode);
    }
    // If all exceptions are of certain type, treat it as TopologyChangedException
    Map<Boolean, List<Entry<Address, Object>>> splitFailures = failures.stream().collect(Collectors.partitioningBy(e -> e.getValue() instanceof CancellationException || e.getValue() instanceof TerminatedWithSnapshotException || isTopologyException((Throwable) e.getValue())));
    List<Entry<Address, Object>> topologyFailures = splitFailures.getOrDefault(true, emptyList());
    List<Entry<Address, Object>> otherFailures = splitFailures.getOrDefault(false, emptyList());
    if (!otherFailures.isEmpty()) {
        return (Throwable) otherFailures.get(0).getValue();
    } else {
        return new TopologyChangedException("Causes from members: " + topologyFailures);
    }
}
Also used : Address(com.hazelcast.cluster.Address) SUSPEND(com.hazelcast.jet.impl.TerminationMode.ActionAfterTerminate.SUSPEND) NOT_RUNNING(com.hazelcast.jet.core.JobStatus.NOT_RUNNING) GetLocalJobMetricsOperation(com.hazelcast.jet.impl.operation.GetLocalJobMetricsOperation) CompletableFuture.completedFuture(java.util.concurrent.CompletableFuture.completedFuture) NonCompletableFuture(com.hazelcast.jet.impl.util.NonCompletableFuture) ExceptionUtil.isTopologyException(com.hazelcast.jet.impl.util.ExceptionUtil.isTopologyException) JobTerminateRequestedException(com.hazelcast.jet.impl.exception.JobTerminateRequestedException) SourceProcessors.readMapP(com.hazelcast.jet.core.processor.SourceProcessors.readMapP) RESTART(com.hazelcast.jet.impl.TerminationMode.ActionAfterTerminate.RESTART) JetDelegatingClassLoader(com.hazelcast.jet.impl.deployment.JetDelegatingClassLoader) TerminatedWithSnapshotException(com.hazelcast.jet.impl.exception.TerminatedWithSnapshotException) Collectors.toMap(java.util.stream.Collectors.toMap) Functions.entryKey(com.hazelcast.function.Functions.entryKey) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) Map(java.util.Map) STARTING(com.hazelcast.jet.core.JobStatus.STARTING) SUSPENDED(com.hazelcast.jet.core.JobStatus.SUSPENDED) DAG(com.hazelcast.jet.core.DAG) JobStatus(com.hazelcast.jet.core.JobStatus) ExceptionUtil(com.hazelcast.jet.impl.util.ExceptionUtil) JobMetrics(com.hazelcast.jet.core.metrics.JobMetrics) CancellationException(java.util.concurrent.CancellationException) CANCEL_GRACEFUL(com.hazelcast.jet.impl.TerminationMode.CANCEL_GRACEFUL) Collections.emptyList(java.util.Collections.emptyList) Collection(java.util.Collection) Set(java.util.Set) UUID(java.util.UUID) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) Collectors(java.util.stream.Collectors) CANCEL_FORCEFUL(com.hazelcast.jet.impl.TerminationMode.CANCEL_FORCEFUL) Objects(java.util.Objects) Util(com.hazelcast.jet.impl.util.Util) List(java.util.List) Util.idToString(com.hazelcast.jet.Util.idToString) ExecutionPlan(com.hazelcast.jet.impl.execution.init.ExecutionPlan) MetricNames(com.hazelcast.jet.core.metrics.MetricNames) Entry(java.util.Map.Entry) TopologyChangedException(com.hazelcast.jet.core.TopologyChangedException) COMPLETED(com.hazelcast.jet.core.JobStatus.COMPLETED) JetDisabledException(com.hazelcast.jet.impl.exception.JetDisabledException) LoggingUtil(com.hazelcast.jet.impl.util.LoggingUtil) ExecutionPlanBuilder.createExecutionPlans(com.hazelcast.jet.impl.execution.init.ExecutionPlanBuilder.createExecutionPlans) Collectors.partitioningBy(java.util.stream.Collectors.partitioningBy) TerminateExecutionOperation(com.hazelcast.jet.impl.operation.TerminateExecutionOperation) ExceptionUtil.isRestartableException(com.hazelcast.jet.impl.util.ExceptionUtil.isRestartableException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) LoggingUtil.logFinest(com.hazelcast.jet.impl.util.LoggingUtil.logFinest) Util.doWithClassLoader(com.hazelcast.jet.impl.util.Util.doWithClassLoader) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) ExecutionService(com.hazelcast.spi.impl.executionservice.ExecutionService) StartExecutionOperation(com.hazelcast.jet.impl.operation.StartExecutionOperation) Function(java.util.function.Function) Supplier(java.util.function.Supplier) Util.formatJobDuration(com.hazelcast.jet.impl.util.Util.formatJobDuration) ActionAfterTerminate(com.hazelcast.jet.impl.TerminationMode.ActionAfterTerminate) ExecutionNotFoundException(com.hazelcast.jet.impl.exception.ExecutionNotFoundException) ArrayList(java.util.ArrayList) JetException(com.hazelcast.jet.JetException) HashSet(java.util.HashSet) InitExecutionOperation(com.hazelcast.jet.impl.operation.InitExecutionOperation) COORDINATOR(com.hazelcast.jet.impl.JobClassLoaderService.JobPhase.COORDINATOR) ILogger(com.hazelcast.logging.ILogger) SnapshotValidator.validateSnapshot(com.hazelcast.jet.impl.SnapshotValidator.validateSnapshot) ExceptionUtil.rethrow(com.hazelcast.jet.impl.util.ExceptionUtil.rethrow) Operation(com.hazelcast.spi.impl.operationservice.Operation) Util.entry(com.hazelcast.jet.Util.entry) ExceptionUtil.withTryCatch(com.hazelcast.jet.impl.util.ExceptionUtil.withTryCatch) BiConsumer(java.util.function.BiConsumer) MembersView(com.hazelcast.internal.cluster.impl.MembersView) LocalMemberResetException(com.hazelcast.core.LocalMemberResetException) RESTART_GRACEFUL(com.hazelcast.jet.impl.TerminationMode.RESTART_GRACEFUL) Edge(com.hazelcast.jet.core.Edge) Version(com.hazelcast.version.Version) EXPORTED_SNAPSHOTS_PREFIX(com.hazelcast.jet.impl.JobRepository.EXPORTED_SNAPSHOTS_PREFIX) Nonnull(javax.annotation.Nonnull) Tuple2(com.hazelcast.jet.datamodel.Tuple2) Nullable(javax.annotation.Nullable) Job(com.hazelcast.jet.Job) Measurement(com.hazelcast.jet.core.metrics.Measurement) SUSPENDED_EXPORTING_SNAPSHOT(com.hazelcast.jet.core.JobStatus.SUSPENDED_EXPORTING_SNAPSHOT) Util.toList(com.hazelcast.jet.impl.util.Util.toList) RawJobMetrics(com.hazelcast.jet.impl.metrics.RawJobMetrics) MetricTags(com.hazelcast.jet.core.metrics.MetricTags) NONE(com.hazelcast.jet.config.ProcessingGuarantee.NONE) Consumer(java.util.function.Consumer) Vertex(com.hazelcast.jet.core.Vertex) Tuple2.tuple2(com.hazelcast.jet.datamodel.Tuple2.tuple2) CustomClassLoadedObject.deserializeWithCustomClassLoader(com.hazelcast.jet.impl.execution.init.CustomClassLoadedObject.deserializeWithCustomClassLoader) ExceptionUtil.peel(com.hazelcast.jet.impl.util.ExceptionUtil.peel) FAILED(com.hazelcast.jet.core.JobStatus.FAILED) RUNNING(com.hazelcast.jet.core.JobStatus.RUNNING) Collections(java.util.Collections) IMap(com.hazelcast.map.IMap) Edge.between(com.hazelcast.jet.core.Edge.between) TerminatedWithSnapshotException(com.hazelcast.jet.impl.exception.TerminatedWithSnapshotException) Address(com.hazelcast.cluster.Address) TopologyChangedException(com.hazelcast.jet.core.TopologyChangedException) Entry(java.util.Map.Entry) CancellationException(java.util.concurrent.CancellationException) Collections.emptyList(java.util.Collections.emptyList) List(java.util.List) ArrayList(java.util.ArrayList) Util.toList(com.hazelcast.jet.impl.util.Util.toList) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) JobTerminateRequestedException(com.hazelcast.jet.impl.exception.JobTerminateRequestedException)

Example 2 with TerminatedWithSnapshotException

use of com.hazelcast.jet.impl.exception.TerminatedWithSnapshotException in project hazelcast by hazelcast.

the class ExecutionContext method beginExecution.

/**
 * Starts local execution of job by submitting tasklets to execution service. If
 * execution was cancelled earlier then execution will not be started.
 * <p>
 * Returns a future which is completed only when all tasklets are completed. If
 * execution was already cancelled before this method is called then the returned
 * future is completed immediately. The future returned can't be cancelled,
 * instead {@link #terminateExecution} should be used.
 */
public CompletableFuture<Void> beginExecution(TaskletExecutionService taskletExecService) {
    synchronized (executionLock) {
        if (executionFuture != null) {
            // beginExecution was already called or execution was cancelled before it started.
            LoggingUtil.logFine(logger, "%s: execution started after cancelled", jobNameAndExecutionId());
            return executionFuture;
        } else {
            // begin job execution
            ClassLoader cl = jetServiceBackend.getJobClassLoaderService().getClassLoader(jobId);
            if (cl == null) {
                cl = nodeEngine.getConfigClassLoader();
            }
            executionFuture = taskletExecService.beginExecute(tasklets, cancellationFuture, cl).whenComplete(withTryCatch(logger, (r, t) -> setCompletionTime())).thenApply(res -> {
                // We ignore this for now.
                if (snapshotContext.isTerminalSnapshot()) {
                    throw new TerminatedWithSnapshotException();
                }
                return res;
            });
            startTime.set(System.currentTimeMillis());
        }
        return executionFuture;
    }
}
Also used : Address(com.hazelcast.cluster.Address) LoggingUtil(com.hazelcast.jet.impl.util.LoggingUtil) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Util.doWithClassLoader(com.hazelcast.jet.impl.util.Util.doWithClassLoader) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) JobTerminateRequestedException(com.hazelcast.jet.impl.exception.JobTerminateRequestedException) Function(java.util.function.Function) MPSCQueue(com.hazelcast.internal.util.concurrent.MPSCQueue) TerminatedWithSnapshotException(com.hazelcast.jet.impl.exception.TerminatedWithSnapshotException) MetricDescriptor(com.hazelcast.internal.metrics.MetricDescriptor) ILogger(com.hazelcast.logging.ILogger) EXECUTION_START_TIME(com.hazelcast.jet.core.metrics.MetricNames.EXECUTION_START_TIME) InternalSerializationService(com.hazelcast.internal.serialization.InternalSerializationService) ExceptionUtil.withTryCatch(com.hazelcast.jet.impl.util.ExceptionUtil.withTryCatch) MwCounter(com.hazelcast.internal.util.counters.MwCounter) Map(java.util.Map) DynamicMetricsProvider(com.hazelcast.internal.metrics.DynamicMetricsProvider) Counter(com.hazelcast.internal.util.counters.Counter) EXECUTION_COMPLETION_TIME(com.hazelcast.jet.core.metrics.MetricNames.EXECUTION_COMPLETION_TIME) ProbeLevel(com.hazelcast.internal.metrics.ProbeLevel) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) VertexDef(com.hazelcast.jet.impl.execution.init.VertexDef) IOUtil(com.hazelcast.internal.nio.IOUtil) NodeEngineImpl(com.hazelcast.spi.impl.NodeEngineImpl) CancellationException(java.util.concurrent.CancellationException) Collections.emptyList(java.util.Collections.emptyList) JobConfig(com.hazelcast.jet.config.JobConfig) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) MetricsCollectionContext(com.hazelcast.internal.metrics.MetricsCollectionContext) Set(java.util.Set) RawJobMetrics(com.hazelcast.jet.impl.metrics.RawJobMetrics) MetricTags(com.hazelcast.jet.core.metrics.MetricTags) File(java.io.File) Objects(java.util.Objects) SnapshotPhase1Result(com.hazelcast.jet.impl.operation.SnapshotPhase1Operation.SnapshotPhase1Result) Util(com.hazelcast.jet.impl.util.Util) List(java.util.List) Util.idToString(com.hazelcast.jet.Util.idToString) TerminationMode(com.hazelcast.jet.impl.TerminationMode) ExecutionPlan(com.hazelcast.jet.impl.execution.init.ExecutionPlan) Entry(java.util.Map.Entry) Collections.unmodifiableMap(java.util.Collections.unmodifiableMap) ProbeUnit(com.hazelcast.internal.metrics.ProbeUnit) Queue(java.util.Queue) JobClassLoaderService(com.hazelcast.jet.impl.JobClassLoaderService) JetServiceBackend(com.hazelcast.jet.impl.JetServiceBackend) TerminatedWithSnapshotException(com.hazelcast.jet.impl.exception.TerminatedWithSnapshotException) Util.doWithClassLoader(com.hazelcast.jet.impl.util.Util.doWithClassLoader)

Aggregations

Address (com.hazelcast.cluster.Address)2 Util.idToString (com.hazelcast.jet.Util.idToString)2 MetricTags (com.hazelcast.jet.core.metrics.MetricTags)2 LocalMemberResetException (com.hazelcast.core.LocalMemberResetException)1 Functions.entryKey (com.hazelcast.function.Functions.entryKey)1 MemberInfo (com.hazelcast.internal.cluster.MemberInfo)1 MembersView (com.hazelcast.internal.cluster.impl.MembersView)1 DynamicMetricsProvider (com.hazelcast.internal.metrics.DynamicMetricsProvider)1 MetricDescriptor (com.hazelcast.internal.metrics.MetricDescriptor)1 MetricsCollectionContext (com.hazelcast.internal.metrics.MetricsCollectionContext)1 ProbeLevel (com.hazelcast.internal.metrics.ProbeLevel)1 ProbeUnit (com.hazelcast.internal.metrics.ProbeUnit)1 IOUtil (com.hazelcast.internal.nio.IOUtil)1 InternalSerializationService (com.hazelcast.internal.serialization.InternalSerializationService)1 MPSCQueue (com.hazelcast.internal.util.concurrent.MPSCQueue)1 Counter (com.hazelcast.internal.util.counters.Counter)1 MwCounter (com.hazelcast.internal.util.counters.MwCounter)1 JetException (com.hazelcast.jet.JetException)1 Job (com.hazelcast.jet.Job)1 Util.entry (com.hazelcast.jet.Util.entry)1