Examples with ComputeJobSibling org.apache.ignite.compute.ComputeJobSibling used on opensource projects

Example 1 with ComputeJobSibling

use of org.apache.ignite.compute.ComputeJobSibling in project ignite by apache.

the class ComputeTaskInternalFuture method finishedFuture.

/**
     * @param ctx Context.
     * @param taskCls Task class.
     * @param e Error.
     * @return Finished task future.
     */
public static <R> ComputeTaskInternalFuture<R> finishedFuture(final GridKernalContext ctx, final Class<?> taskCls, IgniteCheckedException e) {
    assert ctx != null;
    assert taskCls != null;
    assert e != null;
    final long time = U.currentTimeMillis();
    final IgniteUuid id = IgniteUuid.fromUuid(ctx.localNodeId());
    ComputeTaskSession ses = new ComputeTaskSession() {

        @Override
        public String getTaskName() {
            return taskCls.getName();
        }

        @Override
        public UUID getTaskNodeId() {
            return ctx.localNodeId();
        }

        @Override
        public long getStartTime() {
            return time;
        }

        @Override
        public long getEndTime() {
            return time;
        }

        @Override
        public IgniteUuid getId() {
            return id;
        }

        @Override
        public ClassLoader getClassLoader() {
            return null;
        }

        @Override
        public Collection<ComputeJobSibling> getJobSiblings() throws IgniteException {
            return Collections.emptyList();
        }

        @Override
        public Collection<ComputeJobSibling> refreshJobSiblings() throws IgniteException {
            return Collections.emptyList();
        }

        @Nullable
        @Override
        public ComputeJobSibling getJobSibling(IgniteUuid jobId) throws IgniteException {
            return null;
        }

        @Override
        public void setAttribute(Object key, @Nullable Object val) throws IgniteException {
        }

        @Nullable
        @Override
        public <K, V> V getAttribute(K key) {
            return null;
        }

        @Override
        public void setAttributes(Map<?, ?> attrs) throws IgniteException {
        // No-op.
        }

        @Override
        public Map<?, ?> getAttributes() {
            return Collections.emptyMap();
        }

        @Override
        public void addAttributeListener(ComputeTaskSessionAttributeListener lsnr, boolean rewind) {
        // No-op.
        }

        @Override
        public boolean removeAttributeListener(ComputeTaskSessionAttributeListener lsnr) {
            return false;
        }

        @Override
        public <K, V> V waitForAttribute(K key, long timeout) throws InterruptedException {
            throw new InterruptedException("Session was closed.");
        }

        @Override
        public <K, V> boolean waitForAttribute(K key, @Nullable V val, long timeout) throws InterruptedException {
            throw new InterruptedException("Session was closed.");
        }

        @Override
        public Map<?, ?> waitForAttributes(Collection<?> keys, long timeout) throws InterruptedException {
            throw new InterruptedException("Session was closed.");
        }

        @Override
        public boolean waitForAttributes(Map<?, ?> attrs, long timeout) throws InterruptedException {
            throw new InterruptedException("Session was closed.");
        }

        @Override
        public void saveCheckpoint(String key, Object state) {
            throw new IgniteException("Session was closed.");
        }

        @Override
        public void saveCheckpoint(String key, Object state, ComputeTaskSessionScope scope, long timeout) {
            throw new IgniteException("Session was closed.");
        }

        @Override
        public void saveCheckpoint(String key, Object state, ComputeTaskSessionScope scope, long timeout, boolean overwrite) {
            throw new IgniteException("Session was closed.");
        }

        @Nullable
        @Override
        public <T> T loadCheckpoint(String key) throws IgniteException {
            throw new IgniteException("Session was closed.");
        }

        @Override
        public boolean removeCheckpoint(String key) throws IgniteException {
            throw new IgniteException("Session was closed.");
        }

        @Override
        public Collection<UUID> getTopology() {
            return Collections.emptyList();
        }

        @Override
        public IgniteFuture<?> mapFuture() {
            return new IgniteFinishedFutureImpl<Object>();
        }
    };
    ComputeTaskInternalFuture<R> fut = new ComputeTaskInternalFuture<>(ses, ctx);
    fut.onDone(e);
    return fut;
}

Example 2 with ComputeJobSibling

use of org.apache.ignite.compute.ComputeJobSibling in project ignite by apache.

the class GridJobProcessor method processJobExecuteRequest.

/**
     * @param node Node.
     * @param req Request.
     */
@SuppressWarnings("TooBroadScope")
public void processJobExecuteRequest(ClusterNode node, final GridJobExecuteRequest req) {
    if (log.isDebugEnabled())
        log.debug("Received job request message [req=" + req + ", nodeId=" + node.id() + ']');
    PartitionsReservation partsReservation = null;
    if (req.getCacheIds() != null) {
        assert req.getPartition() >= 0 : req;
        assert !F.isEmpty(req.getCacheIds()) : req;
        partsReservation = new PartitionsReservation(req.getCacheIds(), req.getPartition(), req.getTopVer());
    }
    GridJobWorker job = null;
    if (!rwLock.tryReadLock()) {
        if (log.isDebugEnabled())
            log.debug("Received job execution request while stopping this node (will ignore): " + req);
        return;
    }
    try {
        long endTime = req.getCreateTime() + req.getTimeout();
        // Account for overflow.
        if (endTime < 0)
            endTime = Long.MAX_VALUE;
        GridDeployment tmpDep = req.isForceLocalDeployment() ? ctx.deploy().getLocalDeployment(req.getTaskClassName()) : ctx.deploy().getGlobalDeployment(req.getDeploymentMode(), req.getTaskName(), req.getTaskClassName(), req.getUserVersion(), node.id(), req.getClassLoaderId(), req.getLoaderParticipants(), null);
        if (tmpDep == null) {
            if (log.isDebugEnabled())
                log.debug("Checking local tasks...");
            // Check local tasks.
            for (Map.Entry<String, GridDeployment> d : ctx.task().getUsedDeploymentMap().entrySet()) {
                if (d.getValue().classLoaderId().equals(req.getClassLoaderId())) {
                    assert d.getValue().local();
                    tmpDep = d.getValue();
                    break;
                }
            }
        }
        final GridDeployment dep = tmpDep;
        if (log.isDebugEnabled())
            log.debug("Deployment: " + dep);
        boolean releaseDep = true;
        try {
            if (dep != null && dep.acquire()) {
                GridJobSessionImpl jobSes;
                GridJobContextImpl jobCtx;
                try {
                    List<ComputeJobSibling> siblings = null;
                    if (!req.isDynamicSiblings()) {
                        Collection<ComputeJobSibling> siblings0 = req.getSiblings();
                        if (siblings0 == null) {
                            assert req.getSiblingsBytes() != null;
                            siblings0 = U.unmarshal(marsh, req.getSiblingsBytes(), U.resolveClassLoader(ctx.config()));
                        }
                        siblings = new ArrayList<>(siblings0);
                    }
                    Map<Object, Object> sesAttrs = null;
                    if (req.isSessionFullSupport()) {
                        sesAttrs = req.getSessionAttributes();
                        if (sesAttrs == null)
                            sesAttrs = U.unmarshal(marsh, req.getSessionAttributesBytes(), U.resolveClassLoader(dep.classLoader(), ctx.config()));
                    }
                    // Note that we unmarshal session/job attributes here with proper class loader.
                    GridTaskSessionImpl taskSes = ctx.session().createTaskSession(req.getSessionId(), node.id(), req.getTaskName(), dep, req.getTaskClassName(), req.topology(), req.getStartTaskTime(), endTime, siblings, sesAttrs, req.isSessionFullSupport(), req.isInternal(), req.getSubjectId(), req.executorName());
                    taskSes.setCheckpointSpi(req.getCheckpointSpi());
                    taskSes.setClassLoader(dep.classLoader());
                    jobSes = new GridJobSessionImpl(ctx, taskSes, req.getJobId());
                    Map<? extends Serializable, ? extends Serializable> jobAttrs = req.getJobAttributes();
                    if (jobAttrs == null)
                        jobAttrs = U.unmarshal(marsh, req.getJobAttributesBytes(), U.resolveClassLoader(dep.classLoader(), ctx.config()));
                    jobCtx = new GridJobContextImpl(ctx, req.getJobId(), jobAttrs);
                } catch (IgniteCheckedException e) {
                    IgniteException ex = new IgniteException("Failed to deserialize task attributes " + "[taskName=" + req.getTaskName() + ", taskClsName=" + req.getTaskClassName() + ", codeVer=" + req.getUserVersion() + ", taskClsLdr=" + dep.classLoader() + ']', e);
                    U.error(log, ex.getMessage(), e);
                    handleException(node, req, ex, endTime);
                    return;
                }
                job = new GridJobWorker(ctx, dep, req.getCreateTime(), jobSes, jobCtx, req.getJobBytes(), req.getJob(), node, req.isInternal(), evtLsnr, holdLsnr, partsReservation, req.getTopVer(), req.executorName());
                jobCtx.job(job);
                // If exception occurs on job initialization, deployment is released in job listener.
                releaseDep = false;
                if (job.initialize(dep, dep.deployedClass(req.getTaskClassName()))) {
                    // Internal jobs will always be executed synchronously.
                    if (job.isInternal()) {
                        // This is an internal job and can be executed inside busy lock
                        // since job is expected to be short.
                        // This is essential for proper stop without races.
                        job.run();
                        // No execution outside lock.
                        job = null;
                    } else if (jobAlwaysActivate) {
                        if (onBeforeActivateJob(job)) {
                            if (ctx.localNodeId().equals(node.id())) {
                                // Always execute in another thread for local node.
                                executeAsync(job);
                                // No sync execution.
                                job = null;
                            } else if (metricsUpdateFreq > -1L)
                                // Job will be executed synchronously.
                                startedJobsCnt.increment();
                        } else
                            // Job has been cancelled.
                            // Set to null, to avoid sync execution.
                            job = null;
                    } else {
                        GridJobWorker old = passiveJobs.putIfAbsent(job.getJobId(), job);
                        if (old == null)
                            handleCollisions();
                        else
                            U.error(log, "Received computation request with duplicate job ID (could be " + "network malfunction, source node may hang if task timeout was not set) " + "[srcNode=" + node.id() + ", jobId=" + req.getJobId() + ", sesId=" + req.getSessionId() + ", locNodeId=" + ctx.localNodeId() + ']');
                        // No sync execution.
                        job = null;
                    }
                } else
                    // Job was not initialized, no execution.
                    job = null;
            } else {
                // Deployment is null.
                IgniteException ex = new IgniteDeploymentException("Task was not deployed or was redeployed since " + "task execution [taskName=" + req.getTaskName() + ", taskClsName=" + req.getTaskClassName() + ", codeVer=" + req.getUserVersion() + ", clsLdrId=" + req.getClassLoaderId() + ", seqNum=" + req.getClassLoaderId().localId() + ", depMode=" + req.getDeploymentMode() + ", dep=" + dep + ']');
                U.error(log, ex.getMessage(), ex);
                handleException(node, req, ex, endTime);
            }
        } finally {
            if (dep != null && releaseDep)
                release(dep);
        }
    } finally {
        rwLock.readUnlock();
    }
    if (job != null)
        job.run();
}

Example 3 with ComputeJobSibling

use of org.apache.ignite.compute.ComputeJobSibling in project ignite by apache.

the class GridTaskWorker method processMappedJobs.

/**
     * @param jobs Map of jobs.
     * @throws IgniteCheckedException Thrown in case of any error.
     */
private void processMappedJobs(Map<? extends ComputeJob, ClusterNode> jobs) throws IgniteCheckedException {
    if (F.isEmpty(jobs))
        return;
    Collection<GridJobResultImpl> jobResList = new ArrayList<>(jobs.size());
    Collection<ComputeJobSibling> sibs = new ArrayList<>(jobs.size());
    // Map jobs to nodes for computation.
    for (Map.Entry<? extends ComputeJob, ClusterNode> mappedJob : jobs.entrySet()) {
        ComputeJob job = mappedJob.getKey();
        ClusterNode node = mappedJob.getValue();
        if (job == null)
            throw new IgniteCheckedException("Job can not be null [mappedJob=" + mappedJob + ", ses=" + ses + ']');
        if (node == null)
            throw new IgniteCheckedException("Node can not be null [mappedJob=" + mappedJob + ", ses=" + ses + ']');
        IgniteUuid jobId = IgniteUuid.fromUuid(ctx.localNodeId());
        GridJobSiblingImpl sib = new GridJobSiblingImpl(ses.getId(), jobId, node.id(), ctx);
        jobResList.add(new GridJobResultImpl(job, jobId, node, sib));
        // Do not add siblings if result cache is disabled.
        if (resCache)
            sibs.add(sib);
        recordJobEvent(EVT_JOB_MAPPED, jobId, node, "Job got mapped.");
    }
    synchronized (mux) {
        if (state != State.WAITING)
            throw new IgniteCheckedException("Task is not in waiting state [state=" + state + ", ses=" + ses + ']');
        // Do not add siblings if result cache is disabled.
        if (resCache)
            ses.addJobSiblings(sibs);
        if (jobRes == null)
            jobRes = new HashMap<>();
        // getting results while still sending out references.
        for (GridJobResultImpl res : jobResList) {
            if (jobRes.put(res.getJobContext().getJobId(), res) != null)
                throw new IgniteCheckedException("Duplicate job ID for remote job found: " + res.getJobContext().getJobId());
            res.setOccupied(true);
            if (resCache && jobRes.size() > ctx.discovery().size() && jobRes.size() % SPLIT_WARN_THRESHOLD == 0)
                LT.warn(log, "Number of jobs in task is too large for task: " + ses.getTaskName() + ". Consider reducing number of jobs or disabling job result cache with " + "@ComputeTaskNoResultCache annotation.");
        }
    }
    // Set mapped flag.
    ses.onMapped();
    // Send out all remote mappedJobs.
    for (GridJobResultImpl res : jobResList) {
        evtLsnr.onJobSend(this, res.getSibling());
        try {
            sendRequest(res);
        } finally {
            // Open job for processing results.
            synchronized (mux) {
                res.setOccupied(false);
            }
        }
    }
    processDelayedResponses();
}

Example 4 with ComputeJobSibling

use of org.apache.ignite.compute.ComputeJobSibling in project ignite by apache.

the class GridSessionWaitAttributeSelfTest method checkWaitAttributeMethod.

/**
     * @param type Type.
     * @throws Exception If failed.
     */
@SuppressWarnings("unchecked")
private void checkWaitAttributeMethod(WaitAttributeType type) throws Exception {
    assert type != null;
    Ignite ignite1 = G.ignite(getTestIgniteInstanceName() + '1');
    Ignite ignite2 = G.ignite(getTestIgniteInstanceName() + '2');
    assert ignite1 != null;
    assert ignite2 != null;
    ignite1.compute().localDeployTask(TestSessionTask.class, TestSessionTask.class.getClassLoader());
    ComputeTaskFuture<?> fut = ignite1.compute().executeAsync(TestSessionTask.class.getName(), type);
    fut.getTaskSession().mapFuture().get();
    ComputeTaskSession ses = fut.getTaskSession();
    info("Task job siblings [size=" + ses.getJobSiblings().size() + ", siblings=" + ses.getJobSiblings() + ']');
    for (int i = 0; i < ATTR_NUM; i++) {
        String key = createKey("fut", type, i);
        String val = createValue("fut", type, i);
        ses.setAttribute(key, val);
    }
    // Check all job attributes.
    for (ComputeJobSibling sibling : ses.getJobSiblings()) {
        info("Checking session attributes for sibling: " + sibling);
        checkSessionAttributes(ses, sibling.getJobId().toString(), type);
    }
    // Check that fut attributes have been set.
    checkSessionAttributes(ses, "fut", type);
    // Signal finish.
    ses.setAttribute("done", true);
    fut.get();
}

Example 5 with ComputeJobSibling

use of org.apache.ignite.compute.ComputeJobSibling in project ignite by apache.

the class GridSessionCancelSiblingsFromFutureSelfTest method checkTask.

/**
     * @param num Task argument.
     * @throws InterruptedException if failed
     * @throws IgniteCheckedException if failed
     */
private void checkTask(int num) throws InterruptedException, IgniteCheckedException {
    Ignite ignite = G.ignite(getTestIgniteInstanceName());
    ComputeTaskFuture<?> fut = executeAsync(ignite.compute(), GridTaskSessionTestTask.class, num);
    assert fut != null;
    try {
        // Wait until jobs begin execution.
        boolean await = startSig[num].await(WAIT_TIME, TimeUnit.MILLISECONDS);
        assert await : "Jobs did not start.";
        Collection<ComputeJobSibling> jobSiblings = fut.getTaskSession().getJobSiblings();
        // Cancel all jobs.
        for (ComputeJobSibling jobSibling : jobSiblings) {
            jobSibling.cancel();
        }
        Object res = fut.get();
        assert "interrupt-task-data".equals(res) : "Invalid task result: " + res;
        // Wait for all jobs to finish.
        await = stopSig[num].await(WAIT_TIME, TimeUnit.MILLISECONDS);
        assert await : "Jobs did not cancel.";
        int cnt = interruptCnt[num].get();
        assert cnt == SPLIT_COUNT : "Invalid interrupt count value: " + cnt;
    } finally {
        // We must wait for the jobs to be sure that they have completed
        // their execution since they use static variable (shared for the tests).
        fut.get();
    }
}