Examples with JetException com.hazelcast.jet.JetException used on opensource projects

Example 31 with JetException

use of com.hazelcast.jet.JetException in project hazelcast by hazelcast.

the class JobCoordinationService method submitJob.

public CompletableFuture<Void> submitJob(long jobId, Data serializedJobDefinition, JobConfig jobConfig, Subject subject) {
    CompletableFuture<Void> res = new CompletableFuture<>();
    submitToCoordinatorThread(() -> {
        MasterContext masterContext;
        try {
            assertIsMaster("Cannot submit job " + idToString(jobId) + " to non-master node");
            checkOperationalState();
            // the order of operations is important.
            // first, check if the job is already completed
            JobResult jobResult = jobRepository.getJobResult(jobId);
            if (jobResult != null) {
                logger.fine("Not starting job " + idToString(jobId) + " since already completed with result: " + jobResult);
                return;
            }
            if (!config.isResourceUploadEnabled() && !jobConfig.getResourceConfigs().isEmpty()) {
                throw new JetException(Util.JET_RESOURCE_UPLOAD_DISABLED_MESSAGE);
            }
            int quorumSize = jobConfig.isSplitBrainProtectionEnabled() ? getQuorumSize() : 0;
            Object jobDefinition = deserializeJobDefinition(jobId, jobConfig, serializedJobDefinition);
            DAG dag;
            Data serializedDag;
            if (jobDefinition instanceof PipelineImpl) {
                int coopThreadCount = config.getCooperativeThreadCount();
                dag = ((PipelineImpl) jobDefinition).toDag(new Context() {

                    @Override
                    public int defaultLocalParallelism() {
                        return coopThreadCount;
                    }
                });
                serializedDag = nodeEngine().getSerializationService().toData(dag);
            } else {
                dag = (DAG) jobDefinition;
                serializedDag = serializedJobDefinition;
            }
            checkPermissions(subject, dag);
            Set<String> ownedObservables = ownedObservables(dag);
            JobRecord jobRecord = new JobRecord(nodeEngine.getClusterService().getClusterVersion(), jobId, serializedDag, dagToJson(dag), jobConfig, ownedObservables, subject);
            JobExecutionRecord jobExecutionRecord = new JobExecutionRecord(jobId, quorumSize);
            masterContext = createMasterContext(jobRecord, jobExecutionRecord);
            boolean hasDuplicateJobName;
            synchronized (lock) {
                assertIsMaster("Cannot submit job " + idToString(jobId) + " to non-master node");
                checkOperationalState();
                hasDuplicateJobName = jobConfig.getName() != null && hasActiveJobWithName(jobConfig.getName());
                if (!hasDuplicateJobName) {
                    // just try to initiate the coordination
                    MasterContext prev = masterContexts.putIfAbsent(jobId, masterContext);
                    if (prev != null) {
                        logger.fine("Joining to already existing masterContext " + prev.jobIdString());
                        return;
                    }
                }
            }
            if (hasDuplicateJobName) {
                jobRepository.deleteJob(jobId);
                throw new JobAlreadyExistsException("Another active job with equal name (" + jobConfig.getName() + ") exists: " + idToString(jobId));
            }
            // If job is not currently running, it might be that it is just completed
            if (completeMasterContextIfJobAlreadyCompleted(masterContext)) {
                return;
            }
            // If there is no master context and job result at the same time, it means this is the first submission
            jobSubmitted.inc();
            jobRepository.putNewJobRecord(jobRecord);
            logger.info("Starting job " + idToString(masterContext.jobId()) + " based on submit request");
        } catch (Throwable e) {
            jetServiceBackend.getJobClassLoaderService().tryRemoveClassloadersForJob(jobId, COORDINATOR);
            res.completeExceptionally(e);
            throw e;
        } finally {
            res.complete(null);
        }
        tryStartJob(masterContext);
    });
    return res;
}

Example 32 with JetException

use of com.hazelcast.jet.JetException in project hazelcast by hazelcast.

the class MasterJobContext method resolveDagAndCL.

@Nullable
private Tuple2<DAG, ClassLoader> resolveDagAndCL(Supplier<Long> executionIdSupplier) {
    mc.lock();
    try {
        if (isCancelled()) {
            logger.fine("Skipping init job '" + mc.jobName() + "': is already cancelled.");
            throw new CancellationException();
        }
        if (mc.jobStatus() != NOT_RUNNING) {
            logger.fine("Not starting job '" + mc.jobName() + "': status is " + mc.jobStatus());
            return null;
        }
        if (mc.jobExecutionRecord().isSuspended()) {
            mc.jobExecutionRecord().clearSuspended();
            mc.writeJobExecutionRecord(false);
            mc.setJobStatus(NOT_RUNNING);
        }
        if (scheduleRestartIfQuorumAbsent() || scheduleRestartIfClusterIsNotSafe()) {
            return null;
        }
        Version jobClusterVersion = mc.jobRecord().getClusterVersion();
        Version currentClusterVersion = mc.nodeEngine().getClusterService().getClusterVersion();
        if (!jobClusterVersion.equals(currentClusterVersion)) {
            throw new JetException("Cancelling job " + mc.jobName() + ": the cluster was upgraded since the job was " + "submitted. Submitted to version: " + jobClusterVersion + ", current cluster version: " + currentClusterVersion);
        }
        mc.setJobStatus(STARTING);
        // ensure JobExecutionRecord exists
        mc.writeJobExecutionRecord(true);
        if (requestedTerminationMode != null) {
            if (requestedTerminationMode.actionAfterTerminate() != RESTART) {
                throw new JobTerminateRequestedException(requestedTerminationMode);
            }
            // requested termination mode is RESTART, ignore it because we are just starting
            requestedTerminationMode = null;
        }
        ClassLoader classLoader = mc.getJetServiceBackend().getJobClassLoaderService().getOrCreateClassLoader(mc.jobConfig(), mc.jobId(), COORDINATOR);
        DAG dag;
        JobClassLoaderService jobClassLoaderService = mc.getJetServiceBackend().getJobClassLoaderService();
        try {
            jobClassLoaderService.prepareProcessorClassLoaders(mc.jobId());
            dag = deserializeWithCustomClassLoader(mc.nodeEngine().getSerializationService(), classLoader, mc.jobRecord().getDag());
        } catch (Exception e) {
            throw new JetException("DAG deserialization failed", e);
        } finally {
            jobClassLoaderService.clearProcessorClassLoaders();
        }
        // save a copy of the vertex list because it is going to change
        vertices = new HashSet<>();
        dag.iterator().forEachRemaining(vertices::add);
        mc.setExecutionId(executionIdSupplier.get());
        mc.snapshotContext().onExecutionStarted();
        executionCompletionFuture = new CompletableFuture<>();
        return tuple2(dag, classLoader);
    } finally {
        mc.unlock();
    }
}

Example 33 with JetException

use of com.hazelcast.jet.JetException in project hazelcast by hazelcast.

the class StreamJmsP method complete.

@Override
public boolean complete() {
    if (snapshotInProgress) {
        return false;
    }
    while (emitFromTraverser(pendingTraverser)) {
        try {
            Message t = consumer.receiveNoWait();
            if (t == null) {
                pendingTraverser = eventTimeMapper.flatMapIdle();
                break;
            }
            if (guarantee == EXACTLY_ONCE) {
                // execution.
                if (restoredIdsExpiration == Long.MAX_VALUE) {
                    restoredIdsExpiration = System.nanoTime() + RESTORED_IDS_TTL;
                } else if (!restoredIds.isEmpty() && restoredIdsExpiration <= System.nanoTime()) {
                    restoredIds = emptySet();
                }
                Object msgId = messageIdFn.apply(t);
                if (msgId == null) {
                    throw new JetException("Received a message without an ID. All messages must have an ID, " + "you can specify an extracting function using " + JmsSourceBuilder.class.getSimpleName() + ".messageIdFn()");
                }
                seenIds.add(msgId);
                if (restoredIds.remove(msgId)) {
                    logFine(getLogger(), "Redelivered message dropped: %s", t);
                    continue;
                }
            }
            T projectedItem = projectionFn.apply(t);
            pendingTraverser = projectedItem != null ? eventTimeMapper.flatMapEvent(projectedItem, 0, handleJmsTimestamp(t)) : eventTimeMapper.flatMapIdle();
        } catch (JMSException e) {
            throw sneakyThrow(e);
        }
    }
    return false;
}

Example 34 with JetException

use of com.hazelcast.jet.JetException in project hazelcast by hazelcast.

the class StreamFilesP method drainWatcherEvents.

/**
 * @return false, if the watcher should be closed
 */
private boolean drainWatcherEvents() {
    final ILogger logger = getLogger();
    // poll with blocking only when there is no other work to do
    final WatchKey key;
    try {
        key = (currentFile == null && eventQueue.isEmpty()) ? watcher.poll(1, SECONDS) : watcher.poll();
    } catch (InterruptedException e) {
        return false;
    }
    if (key == null) {
        if (!Files.exists(watchedDirectory)) {
            logger.info("Directory " + watchedDirectory + " does not exist, stopped watching");
            return false;
        }
        return true;
    }
    for (WatchEvent<?> event : key.pollEvents()) {
        final WatchEvent.Kind<?> kind = event.kind();
        final Path fileName = ((WatchEvent<Path>) event).context();
        final Path filePath = watchedDirectory.resolve(fileName);
        if (kind == ENTRY_CREATE || kind == ENTRY_MODIFY) {
            if (glob.matches(fileName) && belongsToThisProcessor(fileName) && !Files.isDirectory(filePath)) {
                logFine(logger, "Will open file to read new content: %s", filePath);
                eventQueue.add(filePath);
            }
        } else if (kind == ENTRY_DELETE) {
            logFinest(logger, "File was deleted: %s", filePath);
            fileOffsets.remove(filePath);
        } else if (kind == OVERFLOW) {
            logger.warning("Detected OVERFLOW in " + watchedDirectory);
        } else {
            throw new JetException("Unknown kind of WatchEvent: " + kind);
        }
    }
    if (!key.reset()) {
        logger.info("Watch key is invalid. Stopping watcher.");
        return false;
    }
    return true;
}

Example 35 with JetException

use of com.hazelcast.jet.JetException in project hazelcast by hazelcast.

the class ComputeStageImplBase method attachPeek.

@Nonnull
@SuppressWarnings({ "unchecked", "rawtypes" })
<RET> RET attachPeek(@Nonnull PredicateEx<? super T> shouldLogFn, @Nonnull FunctionEx<? super T, ? extends CharSequence> toStringFn) {
    checkSerializable(shouldLogFn, "shouldLogFn");
    checkSerializable(toStringFn, "toStringFn");
    if (isRebalanceOutput) {
        throw new JetException("peek() not supported after rebalance()");
    }
    return attach(new PeekTransform(transform, fnAdapter.adaptFilterFn(shouldLogFn), fnAdapter.adaptToStringFn(toStringFn)), fnAdapter);
}