Examples with ExecutionStatus org.apache.gobblin.service.ExecutionStatus used on opensource projects

Example 1 with ExecutionStatus

use of org.apache.gobblin.service.ExecutionStatus in project incubator-gobblin by apache.

the class FlowStatusGenerator method getFlowStatus.

/**
 * Get the flow status for a specific execution.
 * @param flowName
 * @param flowGroup
 * @param flowExecutionId
 * @param tag String to filter the returned job statuses
 * @return the flow status, null is returned if the flow status does not exist. If tag is not null, the job status
 * list only contains jobs matching the tag.
 */
public FlowStatus getFlowStatus(String flowName, String flowGroup, long flowExecutionId, String tag) {
    List<JobStatus> jobStatuses = ImmutableList.copyOf(retainStatusOfAnyFlowOrJobMatchingTag(jobStatusRetriever.getJobStatusesForFlowExecution(flowName, flowGroup, flowExecutionId), tag));
    ExecutionStatus flowExecutionStatus = JobStatusRetriever.getFlowStatusFromJobStatuses(jobStatusRetriever.getDagManagerEnabled(), jobStatuses.iterator());
    return jobStatuses.iterator().hasNext() ? new FlowStatus(flowName, flowGroup, flowExecutionId, jobStatuses.iterator(), flowExecutionStatus) : null;
}

Example 2 with ExecutionStatus

use of org.apache.gobblin.service.ExecutionStatus in project incubator-gobblin by apache.

the class FlowStatusGenerator method isFlowRunning.

/**
 * Return true if another instance of a flow is running. A flow is determined to be in the RUNNING state, if any of the
 * jobs in the flow are in the RUNNING state.
 * @param flowName
 * @param flowGroup
 * @return true, if any jobs of the flow are RUNNING.
 */
public boolean isFlowRunning(String flowName, String flowGroup) {
    List<FlowStatus> flowStatusList = getLatestFlowStatus(flowName, flowGroup, 1, null);
    if (flowStatusList == null || flowStatusList.isEmpty()) {
        return false;
    } else {
        FlowStatus flowStatus = flowStatusList.get(0);
        ExecutionStatus flowExecutionStatus = flowStatus.getFlowExecutionStatus();
        return !FINISHED_STATUSES.contains(flowExecutionStatus.name());
    }
}

Example 3 with ExecutionStatus

use of org.apache.gobblin.service.ExecutionStatus in project incubator-gobblin by apache.

the class JobStatusRetriever method getFlowStatusFromJobStatuses.

public static ExecutionStatus getFlowStatusFromJobStatuses(boolean dagManagerEnabled, Iterator<JobStatus> jobStatusIterator) {
    ExecutionStatus flowExecutionStatus = ExecutionStatus.$UNKNOWN;
    if (dagManagerEnabled) {
        while (jobStatusIterator.hasNext()) {
            JobStatus jobStatus = jobStatusIterator.next();
            // Check if this is the flow status instead of a single job status
            if (JobStatusRetriever.isFlowStatus(jobStatus)) {
                flowExecutionStatus = ExecutionStatus.valueOf(jobStatus.getEventName());
            }
        }
    } else {
        Set<ExecutionStatus> jobStatuses = new HashSet<>();
        while (jobStatusIterator.hasNext()) {
            JobStatus jobStatus = jobStatusIterator.next();
            // we actually get and purely calculate flow status based on flow statuses.
            if (!JobStatusRetriever.isFlowStatus(jobStatus)) {
                jobStatuses.add(ExecutionStatus.valueOf(jobStatus.getEventName()));
            }
        }
        List<ExecutionStatus> statusesInDescendingSalience = ImmutableList.of(ExecutionStatus.FAILED, ExecutionStatus.CANCELLED, ExecutionStatus.RUNNING, ExecutionStatus.ORCHESTRATED, ExecutionStatus.COMPLETE);
        flowExecutionStatus = statusesInDescendingSalience.stream().filter(jobStatuses::contains).findFirst().orElse(ExecutionStatus.$UNKNOWN);
    }
    return flowExecutionStatus;
}

Example 4 with ExecutionStatus

use of org.apache.gobblin.service.ExecutionStatus in project incubator-gobblin by apache.

the class JobExecutionPlanListDeserializer method deserialize.

/**
 * Gson invokes this call-back method during deserialization when it encounters a field of the
 * specified type.
 * <p>In the implementation of this call-back method, you should consider invoking
 * {@link JsonDeserializationContext#deserialize(JsonElement, Type)} method to create objects
 * for any non-trivial field of the returned object. However, you should never invoke it on the
 * the same type passing {@code json} since that will cause an infinite loop (Gson will call your
 * call-back method again).
 *
 * @param json The Json data being deserialized
 * @param typeOfT The type of the Object to deserialize to
 * @param context
 * @return a deserialized object of the specified type typeOfT which is a subclass of {@code T}
 * @throws JsonParseException if json is not in the expected format of {@code typeofT}
 */
@Override
public List<JobExecutionPlan> deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
    List<JobExecutionPlan> jobExecutionPlans = new ArrayList<>();
    JsonArray jsonArray = json.getAsJsonArray();
    for (JsonElement jsonElement : jsonArray) {
        JsonObject serializedJobExecutionPlan = (JsonObject) jsonElement;
        JsonObject jobSpecJson = (JsonObject) serializedJobExecutionPlan.get(SerializationConstants.JOB_SPEC_KEY);
        JsonObject specExecutorJson = (JsonObject) serializedJobExecutionPlan.get(SerializationConstants.SPEC_EXECUTOR_KEY);
        ExecutionStatus executionStatus = ExecutionStatus.valueOf(serializedJobExecutionPlan.get(SerializationConstants.EXECUTION_STATUS_KEY).getAsString());
        String uri = jobSpecJson.get(SerializationConstants.JOB_SPEC_URI_KEY).getAsString();
        String version = jobSpecJson.get(SerializationConstants.JOB_SPEC_VERSION_KEY).getAsString();
        String description = jobSpecJson.get(SerializationConstants.JOB_SPEC_DESCRIPTION_KEY).getAsString();
        String templateURI = jobSpecJson.get(SerializationConstants.JOB_SPEC_TEMPLATE_URI_KEY).getAsString();
        String config = jobSpecJson.get(SerializationConstants.JOB_SPEC_CONFIG_KEY).getAsString();
        Config jobConfig = ConfigFactory.parseString(config);
        JobSpec jobSpec;
        try {
            JobSpec.Builder builder = (uri == null) ? JobSpec.builder() : JobSpec.builder(uri);
            builder = (templateURI == null) ? builder : builder.withTemplate(new URI(templateURI));
            builder = (version == null) ? builder : builder.withVersion(version);
            builder = (description == null) ? builder : builder.withDescription(description);
            jobSpec = builder.withConfig(jobConfig).build();
        } catch (URISyntaxException e) {
            log.error("Error deserializing JobSpec {}", config);
            throw new RuntimeException(e);
        }
        Config specExecutorConfig = ConfigFactory.parseString(specExecutorJson.get(SerializationConstants.SPEC_EXECUTOR_CONFIG_KEY).getAsString());
        SpecExecutor specExecutor;
        try {
            URI specExecutorUri = new URI(specExecutorConfig.getString(SerializationConstants.SPEC_EXECUTOR_URI_KEY));
            specExecutor = this.topologySpecMap.get(specExecutorUri).getSpecExecutor();
        } catch (Exception e) {
            log.error("Error deserializing specExecutor {}", specExecutorConfig);
            throw new RuntimeException(e);
        }
        JobExecutionPlan jobExecutionPlan = new JobExecutionPlan(jobSpec, specExecutor);
        jobExecutionPlan.setExecutionStatus(executionStatus);
        JsonElement flowStartTime = serializedJobExecutionPlan.get(SerializationConstants.FLOW_START_TIME_KEY);
        if (flowStartTime != null) {
            jobExecutionPlan.setFlowStartTime(flowStartTime.getAsLong());
        }
        try {
            String jobExecutionFuture = serializedJobExecutionPlan.get(SerializationConstants.JOB_EXECUTION_FUTURE).getAsString();
            Future future = specExecutor.getProducer().get().deserializeAddSpecResponse(jobExecutionFuture);
            jobExecutionPlan.setJobFuture(Optional.fromNullable(future));
        } catch (ExecutionException | InterruptedException e) {
            log.warn("Error during deserialization of JobExecutionFuture.");
            throw new RuntimeException(e);
        }
        jobExecutionPlans.add(jobExecutionPlan);
    }
    return jobExecutionPlans;
}

Example 5 with ExecutionStatus

use of org.apache.gobblin.service.ExecutionStatus in project incubator-gobblin by apache.

the class DagManagerUtils method getNext.

/**
 * Traverse the dag to determine the next set of nodes to be executed. It starts with the startNodes of the dag and
 * identifies each node yet to be executed and for which each of its parent nodes is in the {@link ExecutionStatus#COMPLETE}
 * state.
 */
static Set<DagNode<JobExecutionPlan>> getNext(Dag<JobExecutionPlan> dag) {
    Set<DagNode<JobExecutionPlan>> nextNodesToExecute = new HashSet<>();
    LinkedList<DagNode<JobExecutionPlan>> nodesToExpand = Lists.newLinkedList(dag.getStartNodes());
    FailureOption failureOption = getFailureOption(dag);
    while (!nodesToExpand.isEmpty()) {
        DagNode<JobExecutionPlan> node = nodesToExpand.poll();
        ExecutionStatus executionStatus = getExecutionStatus(node);
        boolean addFlag = true;
        if (executionStatus == ExecutionStatus.PENDING || executionStatus == ExecutionStatus.PENDING_RETRY || executionStatus == ExecutionStatus.PENDING_RESUME) {
            // Add a node to be executed next, only if all of its parent nodes are COMPLETE.
            List<DagNode<JobExecutionPlan>> parentNodes = dag.getParents(node);
            for (DagNode<JobExecutionPlan> parentNode : parentNodes) {
                if (getExecutionStatus(parentNode) != ExecutionStatus.COMPLETE) {
                    addFlag = false;
                    break;
                }
            }
            if (addFlag) {
                nextNodesToExecute.add(node);
            }
        } else if (executionStatus == ExecutionStatus.COMPLETE) {
            // Explore the children of COMPLETED node as next candidates for execution.
            nodesToExpand.addAll(dag.getChildren(node));
        } else if ((executionStatus == ExecutionStatus.FAILED) || (executionStatus == ExecutionStatus.CANCELLED)) {
            switch(failureOption) {
                case FINISH_RUNNING:
                    return new HashSet<>();
                case FINISH_ALL_POSSIBLE:
                default:
                    break;
            }
        }
    }
    return nextNodesToExecute;
}