Examples with ExecutionContext com.hazelcast.jet.impl.execution.ExecutionContext used on opensource projects

Example 6 with ExecutionContext

use of com.hazelcast.jet.impl.execution.ExecutionContext in project hazelcast-jet by hazelcast.

the class JobRestartWithSnapshotTest method getSnapshotContext.

private SnapshotContext getSnapshotContext(Job job) {
    IMapJet<Long, Long> randomIdsMap = instance1.getMap(JobRepository.RANDOM_IDS_MAP_NAME);
    long executionId = randomIdsMap.entrySet().stream().filter(e -> e.getValue().equals(job.getId()) && !e.getValue().equals(e.getKey())).mapToLong(Entry::getKey).findAny().orElseThrow(() -> new AssertionError("ExecutionId not found"));
    ExecutionContext executionContext = null;
    // spin until the executionContext is available on the worker
    while (executionContext == null) {
        executionContext = getJetService(instance2).getJobExecutionService().getExecutionContext(executionId);
    }
    return executionContext.snapshotContext();
}

Example 7 with ExecutionContext

use of com.hazelcast.jet.impl.execution.ExecutionContext in project hazelcast by hazelcast.

the class JobExecutionService method completeExecution.

/**
 * Completes and cleans up execution of the given job
 */
public void completeExecution(@Nonnull ExecutionContext executionContext, Throwable error) {
    ExecutionContext removed = executionContexts.remove(executionContext.executionId());
    if (removed != null) {
        if (error != null) {
            failedJobs.put(executionContext.executionId(), System.nanoTime() + FAILED_EXECUTION_EXPIRY_NS);
        }
        JetDelegatingClassLoader jobClassLoader = jobClassloaderService.getClassLoader(executionContext.jobId());
        try {
            doWithClassLoader(jobClassLoader, () -> executionContext.completeExecution(error));
        } finally {
            if (!executionContext.isLightJob()) {
                jobClassloaderService.tryRemoveClassloadersForJob(executionContext.jobId(), EXECUTION);
            }
            executionCompleted.inc();
            executionContextJobIds.remove(executionContext.jobId());
            logger.fine("Completed execution of " + executionContext.jobNameAndExecutionId());
        }
    }
}

Example 8 with ExecutionContext

use of com.hazelcast.jet.impl.execution.ExecutionContext in project hazelcast by hazelcast.

the class JobExecutionService method checkExecutions.

/**
 * See also javadoc at {@link CheckLightJobsOperation}.
 */
private void checkExecutions() {
    try {
        long now = System.nanoTime();
        long uninitializedContextThreshold = now - UNINITIALIZED_CONTEXT_MAX_AGE_NS;
        Map<Address, List<Long>> executionsPerMember = new HashMap<>();
        for (ExecutionContext ctx : executionContexts.values()) {
            if (!ctx.isLightJob()) {
                continue;
            }
            Address coordinator = ctx.coordinator();
            if (coordinator != null) {
                // if coordinator is known, add execution to the list to check
                executionsPerMember.computeIfAbsent(coordinator, k -> new ArrayList<>()).add(ctx.executionId());
            } else {
                // if coordinator is not known, remove execution if it's not known for too long
                if (ctx.getCreatedOn() <= uninitializedContextThreshold) {
                    LoggingUtil.logFine(logger, "Terminating light job %s because it wasn't initialized during %d seconds", idToString(ctx.executionId()), NANOSECONDS.toSeconds(UNINITIALIZED_CONTEXT_MAX_AGE_NS));
                    terminateExecution0(ctx, TerminationMode.CANCEL_FORCEFUL, new CancellationException());
                }
            }
        }
        // submit the query to the coordinator
        for (Entry<Address, List<Long>> en : executionsPerMember.entrySet()) {
            long[] executionIds = en.getValue().stream().mapToLong(Long::longValue).toArray();
            Operation op = new CheckLightJobsOperation(executionIds);
            InvocationFuture<long[]> future = nodeEngine.getOperationService().createInvocationBuilder(JetServiceBackend.SERVICE_NAME, op, en.getKey()).invoke();
            future.whenComplete((r, t) -> {
                if (t instanceof TargetNotMemberException) {
                    // if the target isn't a member, then all executions are unknown
                    r = executionIds;
                } else if (t != null) {
                    logger.warning("Failed to check light job state with coordinator " + en.getKey() + ": " + t, t);
                    return;
                }
                assert r != null;
                for (long executionId : r) {
                    ExecutionContext execCtx = executionContexts.get(executionId);
                    if (execCtx != null) {
                        logger.fine("Terminating light job " + idToString(executionId) + " because the coordinator doesn't know it");
                        terminateExecution0(execCtx, TerminationMode.CANCEL_FORCEFUL, new CancellationException());
                    }
                }
            });
        }
        // clean up failedJobs
        failedJobs.values().removeIf(expiryTime -> expiryTime < now);
    } catch (Throwable e) {
        logger.severe("Failed to query live light executions: " + e, e);
    }
}

Example 9 with ExecutionContext

use of com.hazelcast.jet.impl.execution.ExecutionContext in project hazelcast by hazelcast.

the class JobExecutionService method initExecution.

/**
 * Initiates the given execution if the local node accepts the coordinator
 * as its master, and has an up-to-date member list information.
 * <ul><li>
 *     If the local node has a stale member list, it retries the init operation
 *     until it receives the new member list from the master.
 * </li><li>
 *     If the local node detects that the member list changed after the init
 *     operation is sent but before executed, then it sends a graceful failure
 *     so that the job init will be retried properly.
 * </li><li>
 *     If there is an already ongoing execution for the given job, then the
 *     init execution is retried.
 * </li></ul>
 */
public void initExecution(long jobId, long executionId, Address coordinator, int coordinatorMemberListVersion, Set<MemberInfo> participants, ExecutionPlan plan) {
    ExecutionContext execCtx = addExecutionContext(jobId, executionId, coordinator, coordinatorMemberListVersion, participants);
    try {
        jobClassloaderService.prepareProcessorClassLoaders(jobId);
        Set<Address> addresses = participants.stream().map(MemberInfo::getAddress).collect(toSet());
        ClassLoader jobCl = jobClassloaderService.getClassLoader(jobId);
        doWithClassLoader(jobCl, () -> execCtx.initialize(coordinator, addresses, plan));
    } finally {
        jobClassloaderService.clearProcessorClassLoaders();
    }
    // initial log entry with all of jobId, jobName, executionId
    logger.info("Execution plan for jobId=" + idToString(jobId) + ", jobName=" + (execCtx.jobName() != null ? '\'' + execCtx.jobName() + '\'' : "null") + ", executionId=" + idToString(executionId) + " initialized");
}

Example 10 with ExecutionContext

use of com.hazelcast.jet.impl.execution.ExecutionContext in project hazelcast by hazelcast.

the class Networking method handleStreamPacket.

private void handleStreamPacket(Packet packet) {
    byte[] payload = packet.toByteArray();
    long executionId = memoryReader.readLong(payload, 0);
    int vertexId = memoryReader.readInt(payload, Long.BYTES);
    int ordinal = memoryReader.readInt(payload, Long.BYTES + Integer.BYTES);
    ExecutionContext executionContext = jobExecutionService.getOrCreateExecutionContext(executionId);
    if (executionContext != null) {
        executionContext.handlePacket(vertexId, ordinal, packet.getConn().getRemoteAddress(), payload);
    }
}