Examples with JobId org.apache.hyracks.api.job.JobId used on opensource projects

Example 21 with JobId

use of org.apache.hyracks.api.job.JobId in project asterixdb by apache.

the class ClientInterfaceIPCI method deliverIncomingMessage.

@Override
public void deliverIncomingMessage(IIPCHandle handle, long mid, long rmid, Object payload, Exception exception) {
    HyracksClientInterfaceFunctions.Function fn = (HyracksClientInterfaceFunctions.Function) payload;
    switch(fn.getFunctionId()) {
        case GET_CLUSTER_CONTROLLER_INFO:
            try {
                handle.send(mid, ccs.getClusterControllerInfo(), null);
            } catch (IPCException e) {
                LOGGER.log(Level.WARNING, "Error sending response to GET_CLUSTER_CONTROLLER_INFO request", e);
            }
            break;
        case GET_JOB_STATUS:
            HyracksClientInterfaceFunctions.GetJobStatusFunction gjsf = (HyracksClientInterfaceFunctions.GetJobStatusFunction) fn;
            ccs.getWorkQueue().schedule(new GetJobStatusWork(ccs.getJobManager(), gjsf.getJobId(), new IPCResponder<>(handle, mid)));
            break;
        case GET_JOB_INFO:
            HyracksClientInterfaceFunctions.GetJobInfoFunction gjif = (HyracksClientInterfaceFunctions.GetJobInfoFunction) fn;
            ccs.getWorkQueue().schedule(new GetJobInfoWork(ccs.getJobManager(), gjif.getJobId(), new IPCResponder<JobInfo>(handle, mid)));
            break;
        case DISTRIBUTE_JOB:
            HyracksClientInterfaceFunctions.DistributeJobFunction djf = (HyracksClientInterfaceFunctions.DistributeJobFunction) fn;
            ccs.getWorkQueue().schedule(new DistributeJobWork(ccs, djf.getACGGFBytes(), jobIdFactory.create(), new IPCResponder<JobId>(handle, mid)));
            break;
        case DESTROY_JOB:
            HyracksClientInterfaceFunctions.DestroyJobFunction dsjf = (HyracksClientInterfaceFunctions.DestroyJobFunction) fn;
            ccs.getWorkQueue().schedule(new DestroyJobWork(ccs, dsjf.getJobId(), new IPCResponder<JobId>(handle, mid)));
            break;
        case CANCEL_JOB:
            HyracksClientInterfaceFunctions.CancelJobFunction cjf = (HyracksClientInterfaceFunctions.CancelJobFunction) fn;
            ccs.getWorkQueue().schedule(new CancelJobWork(ccs.getJobManager(), cjf.getJobId(), new IPCResponder<Void>(handle, mid)));
            break;
        case START_JOB:
            HyracksClientInterfaceFunctions.StartJobFunction sjf = (HyracksClientInterfaceFunctions.StartJobFunction) fn;
            JobId jobId = sjf.getJobId();
            byte[] acggfBytes = null;
            boolean predistributed = false;
            if (jobId == null) {
                //The job is new
                jobId = jobIdFactory.create();
                acggfBytes = sjf.getACGGFBytes();
            } else {
                //The job has been predistributed. We don't need to send an ActivityClusterGraph
                predistributed = true;
            }
            ccs.getWorkQueue().schedule(new JobStartWork(ccs, sjf.getDeploymentId(), acggfBytes, sjf.getJobFlags(), jobId, new IPCResponder<JobId>(handle, mid), predistributed));
            break;
        case GET_DATASET_DIRECTORY_SERIVICE_INFO:
            ccs.getWorkQueue().schedule(new GetDatasetDirectoryServiceInfoWork(ccs, new IPCResponder<NetworkAddress>(handle, mid)));
            break;
        case GET_DATASET_RESULT_STATUS:
            HyracksClientInterfaceFunctions.GetDatasetResultStatusFunction gdrsf = (HyracksClientInterfaceFunctions.GetDatasetResultStatusFunction) fn;
            ccs.getWorkQueue().schedule(new GetResultStatusWork(ccs, gdrsf.getJobId(), gdrsf.getResultSetId(), new IPCResponder<Status>(handle, mid)));
            break;
        case GET_DATASET_RESULT_LOCATIONS:
            HyracksClientInterfaceFunctions.GetDatasetResultLocationsFunction gdrlf = (HyracksClientInterfaceFunctions.GetDatasetResultLocationsFunction) fn;
            ccs.getWorkQueue().schedule(new GetResultPartitionLocationsWork(ccs, gdrlf.getJobId(), gdrlf.getResultSetId(), gdrlf.getKnownRecords(), new IPCResponder<>(handle, mid)));
            break;
        case WAIT_FOR_COMPLETION:
            HyracksClientInterfaceFunctions.WaitForCompletionFunction wfcf = (HyracksClientInterfaceFunctions.WaitForCompletionFunction) fn;
            ccs.getWorkQueue().schedule(new WaitForJobCompletionWork(ccs, wfcf.getJobId(), new IPCResponder<>(handle, mid)));
            break;
        case GET_NODE_CONTROLLERS_INFO:
            ccs.getWorkQueue().schedule(new GetNodeControllersInfoWork(ccs.getNodeManager(), new IPCResponder<>(handle, mid)));
            break;
        case GET_CLUSTER_TOPOLOGY:
            try {
                handle.send(mid, ccs.getCCContext().getClusterTopology(), null);
            } catch (IPCException e) {
                LOGGER.log(Level.WARNING, "Error sending response to GET_CLUSTER_TOPOLOGY request", e);
            }
            break;
        case CLI_DEPLOY_BINARY:
            HyracksClientInterfaceFunctions.CliDeployBinaryFunction dbf = (HyracksClientInterfaceFunctions.CliDeployBinaryFunction) fn;
            ccs.getWorkQueue().schedule(new CliDeployBinaryWork(ccs, dbf.getBinaryURLs(), dbf.getDeploymentId(), new IPCResponder<>(handle, mid)));
            break;
        case CLI_UNDEPLOY_BINARY:
            HyracksClientInterfaceFunctions.CliUnDeployBinaryFunction udbf = (HyracksClientInterfaceFunctions.CliUnDeployBinaryFunction) fn;
            ccs.getWorkQueue().schedule(new CliUnDeployBinaryWork(ccs, udbf.getDeploymentId(), new IPCResponder<>(handle, mid)));
            break;
        case CLUSTER_SHUTDOWN:
            HyracksClientInterfaceFunctions.ClusterShutdownFunction csf = (HyracksClientInterfaceFunctions.ClusterShutdownFunction) fn;
            ccs.getWorkQueue().schedule(new ClusterShutdownWork(ccs, csf.isTerminateNCService(), new IPCResponder<>(handle, mid)));
            break;
        case GET_NODE_DETAILS_JSON:
            HyracksClientInterfaceFunctions.GetNodeDetailsJSONFunction gndjf = (HyracksClientInterfaceFunctions.GetNodeDetailsJSONFunction) fn;
            ccs.getWorkQueue().schedule(new GetNodeDetailsJSONWork(ccs.getNodeManager(), ccs.getCCConfig(), gndjf.getNodeId(), gndjf.isIncludeStats(), gndjf.isIncludeConfig(), new IPCResponder<>(handle, mid)));
            break;
        case THREAD_DUMP:
            HyracksClientInterfaceFunctions.ThreadDumpFunction tdf = (HyracksClientInterfaceFunctions.ThreadDumpFunction) fn;
            ccs.getWorkQueue().schedule(new GetThreadDumpWork(ccs, tdf.getNode(), new IPCResponder<String>(handle, mid)));
            break;
        default:
            try {
                handle.send(mid, null, new IllegalArgumentException("Unknown function " + fn.getFunctionId()));
            } catch (IPCException e) {
                LOGGER.log(Level.WARNING, "Error sending Unknown function response", e);
            }
    }
}

Example 22 with JobId

use of org.apache.hyracks.api.job.JobId in project asterixdb by apache.

the class JobExecutor method abortTaskCluster.

private void abortTaskCluster(TaskClusterAttempt tcAttempt, TaskClusterAttempt.TaskClusterStatus failedOrAbortedStatus) {
    LOGGER.fine("Aborting task cluster: " + tcAttempt.getAttempt());
    Set<TaskAttemptId> abortTaskIds = new HashSet<>();
    Map<String, List<TaskAttemptId>> abortTaskAttemptMap = new HashMap<>();
    for (TaskAttempt ta : tcAttempt.getTaskAttempts().values()) {
        TaskAttemptId taId = ta.getTaskAttemptId();
        TaskAttempt.TaskStatus status = ta.getStatus();
        abortTaskIds.add(taId);
        LOGGER.fine("Checking " + taId + ": " + ta.getStatus());
        if (status == TaskAttempt.TaskStatus.RUNNING || status == TaskAttempt.TaskStatus.COMPLETED) {
            ta.setStatus(TaskAttempt.TaskStatus.ABORTED, null);
            ta.setEndTime(System.currentTimeMillis());
            List<TaskAttemptId> abortTaskAttempts = abortTaskAttemptMap.get(ta.getNodeId());
            if (status == TaskAttempt.TaskStatus.RUNNING && abortTaskAttempts == null) {
                abortTaskAttempts = new ArrayList<>();
                abortTaskAttemptMap.put(ta.getNodeId(), abortTaskAttempts);
            }
            if (status == TaskAttempt.TaskStatus.RUNNING) {
                abortTaskAttempts.add(taId);
            }
        }
    }
    final JobId jobId = jobRun.getJobId();
    LOGGER.fine("Abort map for job: " + jobId + ": " + abortTaskAttemptMap);
    INodeManager nodeManager = ccs.getNodeManager();
    for (Map.Entry<String, List<TaskAttemptId>> entry : abortTaskAttemptMap.entrySet()) {
        final NodeControllerState node = nodeManager.getNodeControllerState(entry.getKey());
        final List<TaskAttemptId> abortTaskAttempts = entry.getValue();
        if (node != null) {
            if (LOGGER.isLoggable(Level.FINE)) {
                LOGGER.fine("Aborting: " + abortTaskAttempts + " at " + entry.getKey());
            }
            try {
                node.getNodeController().abortTasks(jobId, abortTaskAttempts);
            } catch (Exception e) {
                LOGGER.log(Level.SEVERE, e.getMessage(), e);
            }
        }
    }
    inProgressTaskClusters.remove(tcAttempt.getTaskCluster());
    TaskCluster tc = tcAttempt.getTaskCluster();
    PartitionMatchMaker pmm = jobRun.getPartitionMatchMaker();
    pmm.removeUncommittedPartitions(tc.getProducedPartitions(), abortTaskIds);
    pmm.removePartitionRequests(tc.getRequiredPartitions(), abortTaskIds);
    tcAttempt.setStatus(failedOrAbortedStatus);
    tcAttempt.setEndTime(System.currentTimeMillis());
}

Example 23 with JobId

use of org.apache.hyracks.api.job.JobId in project asterixdb by apache.

the class JobManager method finalComplete.

@Override
public void finalComplete(JobRun run) throws HyracksException {
    checkJob(run);
    JobId jobId = run.getJobId();
    HyracksException caughtException = null;
    CCServiceContext serviceCtx = ccs.getContext();
    if (serviceCtx != null) {
        try {
            serviceCtx.notifyJobFinish(jobId);
        } catch (HyracksException e) {
            LOGGER.log(Level.SEVERE, e.getMessage(), e);
            caughtException = e;
        }
    }
    run.setStatus(run.getPendingStatus(), run.getPendingExceptions());
    run.setEndTime(System.currentTimeMillis());
    activeRunMap.remove(jobId);
    runMapArchive.put(jobId, run);
    runMapHistory.put(jobId, run.getExceptions());
    if (run.getActivityClusterGraph().isReportTaskDetails()) {
        /**
             * log job details when profiling is enabled
             */
        try {
            ccs.getJobLogFile().log(createJobLogObject(run));
        } catch (Exception e) {
            LOGGER.log(Level.SEVERE, e.getMessage(), e);
            if (caughtException == null) {
                caughtException = new HyracksException(e);
            } else {
                caughtException.addSuppressed(e);
            }
        }
    }
    // Releases cluster capacitys occupied by the job.
    JobSpecification job = run.getJobSpecification();
    jobCapacityController.release(job);
    // Picks the next job to execute.
    pickJobsToRun();
    // throws caught exceptions if any
    if (caughtException != null) {
        throw caughtException;
    }
}

Example 24 with JobId

use of org.apache.hyracks.api.job.JobId in project asterixdb by apache.

the class JobManager method executeJob.

// Executes a job when the required capacity for the job is met.
private void executeJob(JobRun run) throws HyracksException {
    run.setStartTime(System.currentTimeMillis());
    JobId jobId = run.getJobId();
    activeRunMap.put(jobId, run);
    CCServiceContext serviceCtx = ccs.getContext();
    JobSpecification spec = run.getJobSpecification();
    if (!run.getExecutor().isPredistributed()) {
        serviceCtx.notifyJobCreation(jobId, spec);
    }
    run.setStatus(JobStatus.RUNNING, null);
    executeJobInternal(run);
}

Example 25 with JobId

use of org.apache.hyracks.api.job.JobId in project asterixdb by apache.

the class JobsRESTAPIFunction method invoke.

@Override
public ObjectNode invoke(String host, String servletPath, String[] arguments) throws Exception {
    ObjectMapper om = new ObjectMapper();
    ObjectNode result = om.createObjectNode();
    switch(arguments.length) {
        case 1:
            if (!"".equals(arguments[0])) {
                break;
            }
        case 0:
            {
                GetJobSummariesJSONWork gjse = new GetJobSummariesJSONWork(ccs.getJobManager());
                ccs.getWorkQueue().scheduleAndSync(gjse);
                result.set("result", enhanceSummaries(gjse.getSummaries(), host, servletPath));
                break;
            }
        case 2:
            {
                JobId jobId = JobId.parse(arguments[0]);
                if ("job-activity-graph".equalsIgnoreCase(arguments[1])) {
                    GetActivityClusterGraphJSONWork gjage = new GetActivityClusterGraphJSONWork(ccs, jobId);
                    ccs.getWorkQueue().scheduleAndSync(gjage);
                    result.set("result", gjage.getJSON());
                } else if ("job-run".equalsIgnoreCase(arguments[1])) {
                    GetJobRunJSONWork gjre = new GetJobRunJSONWork(ccs.getJobManager(), jobId);
                    ccs.getWorkQueue().scheduleAndSync(gjre);
                    result.set("result", gjre.getJSON());
                }
                break;
            }
    }
    return result;
}