Search in sources :

Example 1 with ExceptionUtil

use of com.hazelcast.jet.impl.util.ExceptionUtil in project hazelcast-jet by hazelcast.

the class MasterContext method getExecuteResult.

/**
 * <ul>
 * <li>Returns null if there is no failure.
 * <li>Returns CancellationException if the job is cancelled.
 * <li>Returns JobRestartRequestedException if the current execution is cancelled
 * <li>If there is at least one non-restartable failure, such as an exception in user code, then returns that failure.
 * <li>Otherwise, the failure is because a job participant has left the cluster.
 *   In that case, {@code TopologyChangeException} is returned so that the job will be restarted.
 * </ul>
 */
private Throwable getExecuteResult(Map<MemberInfo, Object> responses, boolean isRestartRequested) {
    if (cancellationToken.isCompleted()) {
        logger.fine(jobIdString() + " to be cancelled after execute");
        return new CancellationException();
    } else if (isRestartRequested) {
        return new JobRestartRequestedException();
    }
    Map<Boolean, List<Entry<MemberInfo, Object>>> grouped = groupResponses(responses);
    Collection<MemberInfo> successfulMembers = grouped.get(false).stream().map(Entry::getKey).collect(toList());
    if (successfulMembers.size() == executionPlanMap.size()) {
        logger.fine("Execute of " + jobIdString() + " is successful.");
        return null;
    }
    List<Entry<MemberInfo, Object>> failures = grouped.get(true);
    logger.fine("Execute of " + jobIdString() + " has failures: " + failures);
    // In that case, all remaining participants return a CancellationException.
    return failures.stream().map(e -> (Throwable) e.getValue()).filter(t -> !(t instanceof CancellationException || isTopologicalFailure(t))).findFirst().map(ExceptionUtil::peel).orElse(new TopologyChangedException());
}
Also used : NO_SNAPSHOT(com.hazelcast.jet.impl.execution.SnapshotContext.NO_SNAPSHOT) SnapshotRepository.snapshotDataMapName(com.hazelcast.jet.impl.SnapshotRepository.snapshotDataMapName) NonCompletableFuture(com.hazelcast.jet.impl.util.NonCompletableFuture) Address(com.hazelcast.nio.Address) Util.jobAndExecutionId(com.hazelcast.jet.impl.util.Util.jobAndExecutionId) Processors.mapP(com.hazelcast.jet.core.processor.Processors.mapP) SourceProcessors.readMapP(com.hazelcast.jet.core.processor.SourceProcessors.readMapP) CompletionToken(com.hazelcast.jet.impl.util.CompletionToken) Util.idToString(com.hazelcast.jet.impl.util.Util.idToString) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) Map(java.util.Map) STARTING(com.hazelcast.jet.core.JobStatus.STARTING) DAG(com.hazelcast.jet.core.DAG) JobStatus(com.hazelcast.jet.core.JobStatus) ExceptionUtil(com.hazelcast.jet.impl.util.ExceptionUtil) ExecutionService(com.hazelcast.spi.ExecutionService) Operation(com.hazelcast.spi.Operation) CancellationException(java.util.concurrent.CancellationException) Collections.emptyList(java.util.Collections.emptyList) Collection(java.util.Collection) Util.getJetInstance(com.hazelcast.jet.impl.util.Util.getJetInstance) JobConfig(com.hazelcast.jet.config.JobConfig) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) Collectors(java.util.stream.Collectors) BroadcastKey(com.hazelcast.jet.core.BroadcastKey) List(java.util.List) ExecutionCallback(com.hazelcast.core.ExecutionCallback) ExecutionPlan(com.hazelcast.jet.impl.execution.init.ExecutionPlan) Entry(java.util.Map.Entry) CancelExecutionOperation(com.hazelcast.jet.impl.operation.CancelExecutionOperation) TopologyChangedException(com.hazelcast.jet.core.TopologyChangedException) COMPLETED(com.hazelcast.jet.core.JobStatus.COMPLETED) InternalCompletableFuture(com.hazelcast.spi.InternalCompletableFuture) ExecutionPlanBuilder.createExecutionPlans(com.hazelcast.jet.impl.execution.init.ExecutionPlanBuilder.createExecutionPlans) Collectors.partitioningBy(java.util.stream.Collectors.partitioningBy) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) StartExecutionOperation(com.hazelcast.jet.impl.operation.StartExecutionOperation) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) HashSet(java.util.HashSet) InitExecutionOperation(com.hazelcast.jet.impl.operation.InitExecutionOperation) ILogger(com.hazelcast.logging.ILogger) ExceptionUtil.withTryCatch(com.hazelcast.jet.impl.util.ExceptionUtil.withTryCatch) NOT_STARTED(com.hazelcast.jet.core.JobStatus.NOT_STARTED) MembersView(com.hazelcast.internal.cluster.impl.MembersView) DistributedFunction(com.hazelcast.jet.function.DistributedFunction) Edge(com.hazelcast.jet.core.Edge) ClusterServiceImpl(com.hazelcast.internal.cluster.impl.ClusterServiceImpl) Nullable(javax.annotation.Nullable) NodeEngineImpl(com.hazelcast.spi.impl.NodeEngineImpl) RESTARTING(com.hazelcast.jet.core.JobStatus.RESTARTING) BroadcastEntry(com.hazelcast.jet.impl.execution.BroadcastEntry) ExceptionUtil.isTopologicalFailure(com.hazelcast.jet.impl.util.ExceptionUtil.isTopologicalFailure) DistributedFunctions.entryKey(com.hazelcast.jet.function.DistributedFunctions.entryKey) Consumer(java.util.function.Consumer) Vertex(com.hazelcast.jet.core.Vertex) Collectors.toList(java.util.stream.Collectors.toList) CustomClassLoadedObject.deserializeWithCustomClassLoader(com.hazelcast.jet.impl.execution.init.CustomClassLoadedObject.deserializeWithCustomClassLoader) CompleteExecutionOperation(com.hazelcast.jet.impl.operation.CompleteExecutionOperation) ExceptionUtil.peel(com.hazelcast.jet.impl.util.ExceptionUtil.peel) FAILED(com.hazelcast.jet.core.JobStatus.FAILED) RUNNING(com.hazelcast.jet.core.JobStatus.RUNNING) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) JobRestartRequestedException(com.hazelcast.jet.impl.exception.JobRestartRequestedException) SnapshotOperation(com.hazelcast.jet.impl.operation.SnapshotOperation) Edge.between(com.hazelcast.jet.core.Edge.between) Entry(java.util.Map.Entry) BroadcastEntry(com.hazelcast.jet.impl.execution.BroadcastEntry) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) CancellationException(java.util.concurrent.CancellationException) JobRestartRequestedException(com.hazelcast.jet.impl.exception.JobRestartRequestedException) Collections.emptyList(java.util.Collections.emptyList) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) TopologyChangedException(com.hazelcast.jet.core.TopologyChangedException)

Example 2 with ExceptionUtil

use of com.hazelcast.jet.impl.util.ExceptionUtil in project hazelcast-jet by hazelcast.

the class MasterContext method getInitResult.

/**
 * If there is no failure, then returns null. If the job is cancelled, then returns CancellationException.
 * If there is at least one non-restartable failure, such as an exception in user code, then returns that failure.
 * Otherwise, the failure is because a job participant has left the cluster.
 * In that case, TopologyChangeException is returned so that the job will be restarted.
 */
private Throwable getInitResult(Map<MemberInfo, Object> responses) {
    if (cancellationToken.isCompleted()) {
        logger.fine(jobIdString() + " to be cancelled after init");
        return new CancellationException();
    }
    Map<Boolean, List<Entry<MemberInfo, Object>>> grouped = groupResponses(responses);
    Collection<MemberInfo> successfulMembers = grouped.get(false).stream().map(Entry::getKey).collect(toList());
    if (successfulMembers.size() == executionPlanMap.size()) {
        logger.fine("Init of " + jobIdString() + " is successful.");
        return null;
    }
    List<Entry<MemberInfo, Object>> failures = grouped.get(true);
    logger.fine("Init of " + jobIdString() + " failed with: " + failures);
    // otherwise, return TopologyChangedException so that the job will be restarted
    return failures.stream().map(e -> (Throwable) e.getValue()).filter(t -> !isTopologicalFailure(t)).findFirst().map(ExceptionUtil::peel).orElse(new TopologyChangedException());
}
Also used : NO_SNAPSHOT(com.hazelcast.jet.impl.execution.SnapshotContext.NO_SNAPSHOT) SnapshotRepository.snapshotDataMapName(com.hazelcast.jet.impl.SnapshotRepository.snapshotDataMapName) NonCompletableFuture(com.hazelcast.jet.impl.util.NonCompletableFuture) Address(com.hazelcast.nio.Address) Util.jobAndExecutionId(com.hazelcast.jet.impl.util.Util.jobAndExecutionId) Processors.mapP(com.hazelcast.jet.core.processor.Processors.mapP) SourceProcessors.readMapP(com.hazelcast.jet.core.processor.SourceProcessors.readMapP) CompletionToken(com.hazelcast.jet.impl.util.CompletionToken) Util.idToString(com.hazelcast.jet.impl.util.Util.idToString) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) Map(java.util.Map) STARTING(com.hazelcast.jet.core.JobStatus.STARTING) DAG(com.hazelcast.jet.core.DAG) JobStatus(com.hazelcast.jet.core.JobStatus) ExceptionUtil(com.hazelcast.jet.impl.util.ExceptionUtil) ExecutionService(com.hazelcast.spi.ExecutionService) Operation(com.hazelcast.spi.Operation) CancellationException(java.util.concurrent.CancellationException) Collections.emptyList(java.util.Collections.emptyList) Collection(java.util.Collection) Util.getJetInstance(com.hazelcast.jet.impl.util.Util.getJetInstance) JobConfig(com.hazelcast.jet.config.JobConfig) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) Collectors(java.util.stream.Collectors) BroadcastKey(com.hazelcast.jet.core.BroadcastKey) List(java.util.List) ExecutionCallback(com.hazelcast.core.ExecutionCallback) ExecutionPlan(com.hazelcast.jet.impl.execution.init.ExecutionPlan) Entry(java.util.Map.Entry) CancelExecutionOperation(com.hazelcast.jet.impl.operation.CancelExecutionOperation) TopologyChangedException(com.hazelcast.jet.core.TopologyChangedException) COMPLETED(com.hazelcast.jet.core.JobStatus.COMPLETED) InternalCompletableFuture(com.hazelcast.spi.InternalCompletableFuture) ExecutionPlanBuilder.createExecutionPlans(com.hazelcast.jet.impl.execution.init.ExecutionPlanBuilder.createExecutionPlans) Collectors.partitioningBy(java.util.stream.Collectors.partitioningBy) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) StartExecutionOperation(com.hazelcast.jet.impl.operation.StartExecutionOperation) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) HashSet(java.util.HashSet) InitExecutionOperation(com.hazelcast.jet.impl.operation.InitExecutionOperation) ILogger(com.hazelcast.logging.ILogger) ExceptionUtil.withTryCatch(com.hazelcast.jet.impl.util.ExceptionUtil.withTryCatch) NOT_STARTED(com.hazelcast.jet.core.JobStatus.NOT_STARTED) MembersView(com.hazelcast.internal.cluster.impl.MembersView) DistributedFunction(com.hazelcast.jet.function.DistributedFunction) Edge(com.hazelcast.jet.core.Edge) ClusterServiceImpl(com.hazelcast.internal.cluster.impl.ClusterServiceImpl) Nullable(javax.annotation.Nullable) NodeEngineImpl(com.hazelcast.spi.impl.NodeEngineImpl) RESTARTING(com.hazelcast.jet.core.JobStatus.RESTARTING) BroadcastEntry(com.hazelcast.jet.impl.execution.BroadcastEntry) ExceptionUtil.isTopologicalFailure(com.hazelcast.jet.impl.util.ExceptionUtil.isTopologicalFailure) DistributedFunctions.entryKey(com.hazelcast.jet.function.DistributedFunctions.entryKey) Consumer(java.util.function.Consumer) Vertex(com.hazelcast.jet.core.Vertex) Collectors.toList(java.util.stream.Collectors.toList) CustomClassLoadedObject.deserializeWithCustomClassLoader(com.hazelcast.jet.impl.execution.init.CustomClassLoadedObject.deserializeWithCustomClassLoader) CompleteExecutionOperation(com.hazelcast.jet.impl.operation.CompleteExecutionOperation) ExceptionUtil.peel(com.hazelcast.jet.impl.util.ExceptionUtil.peel) FAILED(com.hazelcast.jet.core.JobStatus.FAILED) RUNNING(com.hazelcast.jet.core.JobStatus.RUNNING) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) JobRestartRequestedException(com.hazelcast.jet.impl.exception.JobRestartRequestedException) SnapshotOperation(com.hazelcast.jet.impl.operation.SnapshotOperation) Edge.between(com.hazelcast.jet.core.Edge.between) Entry(java.util.Map.Entry) BroadcastEntry(com.hazelcast.jet.impl.execution.BroadcastEntry) MemberInfo(com.hazelcast.internal.cluster.MemberInfo) CancellationException(java.util.concurrent.CancellationException) Collections.emptyList(java.util.Collections.emptyList) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) TopologyChangedException(com.hazelcast.jet.core.TopologyChangedException)

Aggregations

ExecutionCallback (com.hazelcast.core.ExecutionCallback)2 MemberInfo (com.hazelcast.internal.cluster.MemberInfo)2 ClusterServiceImpl (com.hazelcast.internal.cluster.impl.ClusterServiceImpl)2 MembersView (com.hazelcast.internal.cluster.impl.MembersView)2 JobConfig (com.hazelcast.jet.config.JobConfig)2 ProcessingGuarantee (com.hazelcast.jet.config.ProcessingGuarantee)2 BroadcastKey (com.hazelcast.jet.core.BroadcastKey)2 DAG (com.hazelcast.jet.core.DAG)2 Edge (com.hazelcast.jet.core.Edge)2 Edge.between (com.hazelcast.jet.core.Edge.between)2 JobStatus (com.hazelcast.jet.core.JobStatus)2 COMPLETED (com.hazelcast.jet.core.JobStatus.COMPLETED)2 FAILED (com.hazelcast.jet.core.JobStatus.FAILED)2 NOT_STARTED (com.hazelcast.jet.core.JobStatus.NOT_STARTED)2 RESTARTING (com.hazelcast.jet.core.JobStatus.RESTARTING)2 RUNNING (com.hazelcast.jet.core.JobStatus.RUNNING)2 STARTING (com.hazelcast.jet.core.JobStatus.STARTING)2 TopologyChangedException (com.hazelcast.jet.core.TopologyChangedException)2 Vertex (com.hazelcast.jet.core.Vertex)2 Processors.mapP (com.hazelcast.jet.core.processor.Processors.mapP)2