Examples with ProgramRunStatus io.cdap.cdap.proto.ProgramRunStatus used on opensource projects

Example 26 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by cdapio.

the class ProgramStatusEventPublisher method processMessages.

@Override
protected void processMessages(StructuredTableContext structuredTableContext, Iterator<ImmutablePair<String, Notification>> messages) {
    List<ProgramStatusEvent> programStatusEvents = new ArrayList<>();
    long publishTime = System.currentTimeMillis();
    messages.forEachRemaining(message -> {
        Notification notification = message.getSecond();
        if (!notification.getNotificationType().equals(Notification.Type.PROGRAM_STATUS)) {
            return;
        }
        Map<String, String> properties = notification.getProperties();
        // get program run ID
        String programStatus = properties.get(ProgramOptionConstants.PROGRAM_STATUS);
        if (programStatus == null) {
            return;
        }
        ProgramRunStatus programRunStatus = ProgramRunStatus.valueOf(programStatus);
        String programRun = properties.get(ProgramOptionConstants.PROGRAM_RUN_ID);
        ProgramRunId programRunId = GSON.fromJson(programRun, ProgramRunId.class);
        // Should event publish happen for this status
        if (!shouldPublish(programRunId)) {
            return;
        }
        ProgramStatusEventDetails.Builder builder = ProgramStatusEventDetails.getBuilder(programRunId.getRun(), programRunId.getApplication(), programRunId.getProgram(), programRunId.getNamespace(), programStatus, RunIds.getTime(programRunId.getRun(), TimeUnit.MILLISECONDS));
        String userArgsString = properties.get(ProgramOptionConstants.USER_OVERRIDES);
        String sysArgsString = properties.get(ProgramOptionConstants.SYSTEM_OVERRIDES);
        Type argsMapType = new TypeToken<Map<String, String>>() {
        }.getType();
        builder = builder.withUserArgs(GSON.fromJson(userArgsString, argsMapType)).withSystemArgs(GSON.fromJson(sysArgsString, argsMapType));
        if (programRunStatus.isEndState()) {
            builder = populateErrorDetailsAndMetrics(builder, properties, programRunStatus, programRunId);
        }
        ProgramStatusEventDetails programStatusEventDetails = builder.build();
        ProgramStatusEvent programStatusEvent = new ProgramStatusEvent(publishTime, EVENT_VERSION, instanceName, projectName, programStatusEventDetails);
        programStatusEvents.add(programStatusEvent);
    });
    this.eventWriters.forEach(eventWriter -> eventWriter.write(programStatusEvents));
}

Example 27 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by cdapio.

the class ProgramLifecycleHttpHandler method programHistory.

/**
 * Returns program runs of an app version based on options it returns either currently running or completed or failed.
 * Default it returns all.
 */
@GET
@Path("/apps/{app-name}/versions/{app-version}/{program-type}/{program-name}/runs")
public void programHistory(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("app-name") String appName, @PathParam("app-version") String appVersion, @PathParam("program-type") String type, @PathParam("program-name") String programName, @QueryParam("status") String status, @QueryParam("start") String startTs, @QueryParam("end") String endTs, @QueryParam("limit") @DefaultValue("100") final int resultLimit) throws Exception {
    ProgramType programType = getProgramType(type);
    long start = (startTs == null || startTs.isEmpty()) ? 0 : Long.parseLong(startTs);
    long end = (endTs == null || endTs.isEmpty()) ? Long.MAX_VALUE : Long.parseLong(endTs);
    ProgramId program = new ApplicationId(namespaceId, appName, appVersion).program(programType, programName);
    ProgramRunStatus runStatus = (status == null) ? ProgramRunStatus.ALL : ProgramRunStatus.valueOf(status.toUpperCase());
    List<RunRecord> records = lifecycleService.getRunRecords(program, runStatus, start, end, resultLimit).stream().filter(record -> !isTetheredRunRecord(record)).collect(Collectors.toList());
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(records));
}

Example 28 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by cdapio.

the class ProgramLifecycleService method getProgramStatus.

/**
 * Returns the program status based on the active run records of a program.
 * A program is RUNNING if there are any RUNNING or SUSPENDED run records.
 * A program is starting if there are any PENDING or STARTING run records and no RUNNING run records.
 * Otherwise, it is STOPPED.
 *
 * @param runRecords run records for the program
 * @return the program status
 */
@VisibleForTesting
static ProgramStatus getProgramStatus(Collection<RunRecordDetail> runRecords) {
    boolean hasStarting = false;
    for (RunRecordDetail runRecord : runRecords) {
        ProgramRunStatus runStatus = runRecord.getStatus();
        if (runStatus == ProgramRunStatus.RUNNING || runStatus == ProgramRunStatus.SUSPENDED) {
            return ProgramStatus.RUNNING;
        }
        hasStarting = hasStarting || runStatus == ProgramRunStatus.STARTING || runStatus == ProgramRunStatus.PENDING;
    }
    return hasStarting ? ProgramStatus.STARTING : ProgramStatus.STOPPED;
}

Example 29 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by cdapio.

the class ProgramNotificationSubscriberService method processWorkflowOnStop.

/**
 * On workflow program stop, inspects inner program states and adjust them if they are not in end state already.
 *
 * @param appMetadataStore the {@link AppMetadataStore} to write the status to
 * @param programHeartbeatTable the {@link ProgramHeartbeatTable} to write the status to
 * @param programRunId the program run of the completed program
 * @param programRunStatus the status of the completion
 * @param notification the {@link Notification} that carries information about the workflow completion
 * @param sourceId the source message id of the notification
 * @param runnables a {@link List} adding {@link Runnable} to be executed after event handling is completed
 * @throws Exception if failed to update program status
 */
private void processWorkflowOnStop(AppMetadataStore appMetadataStore, ProgramHeartbeatTable programHeartbeatTable, ProgramRunId programRunId, ProgramRunStatus programRunStatus, Notification notification, byte[] sourceId, List<Runnable> runnables) throws Exception {
    ApplicationId appId = programRunId.getParent().getParent();
    WorkflowSpecification workflowSpec = Optional.ofNullable(appMetadataStore.getApplication(appId)).map(appMeta -> appMeta.getSpec().getWorkflows().get(programRunId.getProgram())).orElse(null);
    // If cannot find the workflow spec (e.g. app deleted), then there is nothing we can do.
    if (workflowSpec == null) {
        return;
    }
    // For all MR and Spark nodes, we need to update the inner program run status if they are not in end state yet.
    for (WorkflowNode workflowNode : workflowSpec.getNodeIdMap().values()) {
        if (!(workflowNode instanceof WorkflowActionNode)) {
            continue;
        }
        ScheduleProgramInfo programInfo = ((WorkflowActionNode) workflowNode).getProgram();
        if (!WORKFLOW_INNER_PROGRAM_TYPES.containsKey(programInfo.getProgramType())) {
            continue;
        }
        // Get all active runs of the inner program. If the parent workflow runId is the same as this one,
        // set a terminal state for the inner program run.
        ProgramId innerProgramId = appId.program(WORKFLOW_INNER_PROGRAM_TYPES.get(programInfo.getProgramType()), programInfo.getProgramName());
        Map<ProgramRunId, Notification> innerProgramNotifications = new LinkedHashMap<>();
        appMetadataStore.scanActiveRuns(innerProgramId, runRecord -> {
            Map<String, String> systemArgs = runRecord.getSystemArgs();
            String workflowName = systemArgs.get(ProgramOptionConstants.WORKFLOW_NAME);
            String workflowRun = systemArgs.get(ProgramOptionConstants.WORKFLOW_RUN_ID);
            if (workflowName == null || workflowRun == null) {
                return;
            }
            ProgramRunId workflowRunId = appId.program(ProgramType.WORKFLOW, workflowName).run(workflowRun);
            if (!programRunId.equals(workflowRunId)) {
                return;
            }
            Map<String, String> notificationProps = new HashMap<>(notification.getProperties());
            notificationProps.put(ProgramOptionConstants.PROGRAM_RUN_ID, GSON.toJson(runRecord.getProgramRunId()));
            innerProgramNotifications.put(runRecord.getProgramRunId(), new Notification(Notification.Type.PROGRAM_STATUS, notificationProps));
        });
        for (Map.Entry<ProgramRunId, Notification> entry : innerProgramNotifications.entrySet()) {
            handleProgramEvent(entry.getKey(), programRunStatus, entry.getValue(), sourceId, appMetadataStore, programHeartbeatTable, runnables);
        }
    }
}

Example 30 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by cdapio.

the class RunRecordCorrectorService method doFixRunRecords.

/**
 * Fix all the possible inconsistent states for RunRecords that shows it is in RUNNING state but actually not
 * via check to {@link ProgramRuntimeService} for a type of CDAP program.
 *
 * @return the set of fixed {@link ProgramRunId}.
 */
private Set<ProgramRunId> doFixRunRecords() {
    LOG.trace("Start getting run records not actually running ...");
    // Get run records in STARTING, RUNNING and SUSPENDED states that are actually not running
    // Do it in micro batches of transactions to avoid tx timeout
    Set<ProgramRunId> fixedPrograms = new HashSet<>();
    Predicate<RunRecordDetail> filter = createFilter(fixedPrograms);
    for (ProgramRunStatus status : NOT_STOPPED_STATUSES) {
        while (true) {
            // runs are not guaranteed to come back in order of start time, so need to scan the entire time range
            // each time. Should not be worse in performance than specifying a more restrictive time range
            // because time range is just used as a read-time filter.
            Map<ProgramRunId, RunRecordDetail> runs = store.getRuns(status, 0L, Long.MAX_VALUE, txBatchSize, filter);
            LOG.trace("{} run records in {} state but are not actually running", runs.size(), status);
            if (runs.isEmpty()) {
                break;
            }
            for (RunRecordDetail record : runs.values()) {
                ProgramRunId programRunId = record.getProgramRunId();
                String msg = String.format("Fixed RunRecord for program run %s in %s state because it is actually not running", programRunId, record.getStatus());
                programStateWriter.error(programRunId, new ProgramRunAbortedException(msg));
                fixedPrograms.add(programRunId);
                LOG.warn(msg);
            }
        }
    }
    if (fixedPrograms.isEmpty()) {
        LOG.trace("No RunRecord found with status in {}, but the program are not actually running", NOT_STOPPED_STATUSES);
    } else {
        LOG.warn("Fixed {} RunRecords with status in {}, but the programs are not actually running", fixedPrograms.size(), NOT_STOPPED_STATUSES);
    }
    return fixedPrograms;
}