Examples with ProgramRunStatus io.cdap.cdap.proto.ProgramRunStatus used on opensource projects

Example 16 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by caskdata.

the class ProgramLifecycleService method getActiveRuns.

/**
 * Returns the active run records (PENDING / STARTING / RUNNING / SUSPENDED) based on the given program id and an
 * optional run id.
 */
private Map<ProgramRunId, RunRecordDetail> getActiveRuns(ProgramId programId, @Nullable String runId) {
    if (runId == null) {
        return store.getActiveRuns(programId);
    }
    RunRecordDetail runRecord = store.getRun(programId.run(runId));
    EnumSet<ProgramRunStatus> activeStates = EnumSet.of(ProgramRunStatus.PENDING, ProgramRunStatus.STARTING, ProgramRunStatus.RUNNING, ProgramRunStatus.SUSPENDED);
    return runRecord == null || !activeStates.contains(runRecord.getStatus()) ? Collections.emptyMap() : Collections.singletonMap(programId.run(runId), runRecord);
}

Example 17 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by caskdata.

the class ProgramLifecycleHttpHandler method programHistory.

/**
 * Returns program runs of an app version based on options it returns either currently running or completed or failed.
 * Default it returns all.
 */
@GET
@Path("/apps/{app-name}/versions/{app-version}/{program-type}/{program-name}/runs")
public void programHistory(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("app-name") String appName, @PathParam("app-version") String appVersion, @PathParam("program-type") String type, @PathParam("program-name") String programName, @QueryParam("status") String status, @QueryParam("start") String startTs, @QueryParam("end") String endTs, @QueryParam("limit") @DefaultValue("100") final int resultLimit) throws Exception {
    ProgramType programType = getProgramType(type);
    long start = (startTs == null || startTs.isEmpty()) ? 0 : Long.parseLong(startTs);
    long end = (endTs == null || endTs.isEmpty()) ? Long.MAX_VALUE : Long.parseLong(endTs);
    ProgramId program = new ApplicationId(namespaceId, appName, appVersion).program(programType, programName);
    ProgramRunStatus runStatus = (status == null) ? ProgramRunStatus.ALL : ProgramRunStatus.valueOf(status.toUpperCase());
    List<RunRecord> records = lifecycleService.getRunRecords(program, runStatus, start, end, resultLimit).stream().filter(record -> !isTetheredRunRecord(record)).collect(Collectors.toList());
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(records));
}

Example 18 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by caskdata.

the class ProgramNotificationSubscriberService method processNotification.

/**
 * Process a {@link Notification} received from TMS.
 *
 * @param programHeartbeatTable the {@link ProgramHeartbeatTable} for writing heart beats and program status
 * @param messageIdBytes the raw message id in the TMS for the notification
 * @param notification the {@link Notification} to process
 * @param context context to get the table for operations
 * @return a {@link List} of {@link Runnable} tasks to run after the transactional processing of the whole
 *         messages batch is completed
 * @throws Exception if failed to process the given notification
 */
private List<Runnable> processNotification(ProgramHeartbeatTable programHeartbeatTable, byte[] messageIdBytes, Notification notification, StructuredTableContext context) throws Exception {
    AppMetadataStore appMetadataStore = AppMetadataStore.create(context);
    Map<String, String> properties = notification.getProperties();
    // Required parameters
    String programRun = properties.get(ProgramOptionConstants.PROGRAM_RUN_ID);
    String programStatusStr = properties.get(ProgramOptionConstants.PROGRAM_STATUS);
    String clusterStatusStr = properties.get(ProgramOptionConstants.CLUSTER_STATUS);
    // Ignore notifications which specify an invalid ProgramRunId, which shouldn't happen
    if (programRun == null) {
        LOG.warn("Ignore notification that misses program run state information, {}", notification);
        return Collections.emptyList();
    }
    ProgramRunId programRunId = GSON.fromJson(programRun, ProgramRunId.class);
    ProgramRunStatus programRunStatus = null;
    if (programStatusStr != null) {
        try {
            programRunStatus = ProgramRunStatus.valueOf(programStatusStr);
        } catch (IllegalArgumentException e) {
            LOG.warn("Ignore notification with invalid program run status {} for program {}, {}", programStatusStr, programRun, notification);
            return Collections.emptyList();
        }
    }
    ProgramRunClusterStatus clusterStatus = null;
    if (clusterStatusStr != null) {
        try {
            clusterStatus = ProgramRunClusterStatus.valueOf(clusterStatusStr);
        } catch (IllegalArgumentException e) {
            LOG.warn("Ignore notification with invalid program run cluster status {} for program {}", clusterStatusStr, programRun);
            return Collections.emptyList();
        }
    }
    if (notification.getNotificationType().equals(Notification.Type.PROGRAM_HEART_BEAT)) {
        RunRecordDetail runRecordMeta = appMetadataStore.getRun(programRunId);
        long heartBeatTimeInSeconds = TimeUnit.MILLISECONDS.toSeconds(Long.parseLong(properties.get(ProgramOptionConstants.HEART_BEAT_TIME)));
        writeToHeartBeatTable(runRecordMeta, heartBeatTimeInSeconds, programHeartbeatTable);
        // we can return after writing to heart beat table
        return Collections.emptyList();
    }
    List<Runnable> result = new ArrayList<>();
    if (programRunStatus != null) {
        handleProgramEvent(programRunId, programRunStatus, notification, messageIdBytes, appMetadataStore, programHeartbeatTable, result);
    }
    if (clusterStatus == null) {
        return result;
    }
    handleClusterEvent(programRunId, clusterStatus, notification, messageIdBytes, appMetadataStore, context).ifPresent(result::add);
    return result;
}

Example 19 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by caskdata.

the class ProgramNotificationSubscriberService method processWorkflowOnStop.

/**
 * On workflow program stop, inspects inner program states and adjust them if they are not in end state already.
 *
 * @param appMetadataStore the {@link AppMetadataStore} to write the status to
 * @param programHeartbeatTable the {@link ProgramHeartbeatTable} to write the status to
 * @param programRunId the program run of the completed program
 * @param programRunStatus the status of the completion
 * @param notification the {@link Notification} that carries information about the workflow completion
 * @param sourceId the source message id of the notification
 * @param runnables a {@link List} adding {@link Runnable} to be executed after event handling is completed
 * @throws Exception if failed to update program status
 */
private void processWorkflowOnStop(AppMetadataStore appMetadataStore, ProgramHeartbeatTable programHeartbeatTable, ProgramRunId programRunId, ProgramRunStatus programRunStatus, Notification notification, byte[] sourceId, List<Runnable> runnables) throws Exception {
    ApplicationId appId = programRunId.getParent().getParent();
    WorkflowSpecification workflowSpec = Optional.ofNullable(appMetadataStore.getApplication(appId)).map(appMeta -> appMeta.getSpec().getWorkflows().get(programRunId.getProgram())).orElse(null);
    // If cannot find the workflow spec (e.g. app deleted), then there is nothing we can do.
    if (workflowSpec == null) {
        return;
    }
    // For all MR and Spark nodes, we need to update the inner program run status if they are not in end state yet.
    for (WorkflowNode workflowNode : workflowSpec.getNodeIdMap().values()) {
        if (!(workflowNode instanceof WorkflowActionNode)) {
            continue;
        }
        ScheduleProgramInfo programInfo = ((WorkflowActionNode) workflowNode).getProgram();
        if (!WORKFLOW_INNER_PROGRAM_TYPES.containsKey(programInfo.getProgramType())) {
            continue;
        }
        // Get all active runs of the inner program. If the parent workflow runId is the same as this one,
        // set a terminal state for the inner program run.
        ProgramId innerProgramId = appId.program(WORKFLOW_INNER_PROGRAM_TYPES.get(programInfo.getProgramType()), programInfo.getProgramName());
        Map<ProgramRunId, Notification> innerProgramNotifications = new LinkedHashMap<>();
        appMetadataStore.scanActiveRuns(innerProgramId, runRecord -> {
            Map<String, String> systemArgs = runRecord.getSystemArgs();
            String workflowName = systemArgs.get(ProgramOptionConstants.WORKFLOW_NAME);
            String workflowRun = systemArgs.get(ProgramOptionConstants.WORKFLOW_RUN_ID);
            if (workflowName == null || workflowRun == null) {
                return;
            }
            ProgramRunId workflowRunId = appId.program(ProgramType.WORKFLOW, workflowName).run(workflowRun);
            if (!programRunId.equals(workflowRunId)) {
                return;
            }
            Map<String, String> notificationProps = new HashMap<>(notification.getProperties());
            notificationProps.put(ProgramOptionConstants.PROGRAM_RUN_ID, GSON.toJson(runRecord.getProgramRunId()));
            innerProgramNotifications.put(runRecord.getProgramRunId(), new Notification(Notification.Type.PROGRAM_STATUS, notificationProps));
        });
        for (Map.Entry<ProgramRunId, Notification> entry : innerProgramNotifications.entrySet()) {
            handleProgramEvent(entry.getKey(), programRunStatus, entry.getValue(), sourceId, appMetadataStore, programHeartbeatTable, runnables);
        }
    }
}

Example 20 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by caskdata.

the class ProgramStatusEventPublisher method processMessages.

@Override
protected void processMessages(StructuredTableContext structuredTableContext, Iterator<ImmutablePair<String, Notification>> messages) {
    List<ProgramStatusEvent> programStatusEvents = new ArrayList<>();
    long publishTime = System.currentTimeMillis();
    messages.forEachRemaining(message -> {
        Notification notification = message.getSecond();
        if (!notification.getNotificationType().equals(Notification.Type.PROGRAM_STATUS)) {
            return;
        }
        Map<String, String> properties = notification.getProperties();
        // get program run ID
        String programStatus = properties.get(ProgramOptionConstants.PROGRAM_STATUS);
        if (programStatus == null) {
            return;
        }
        ProgramRunStatus programRunStatus = ProgramRunStatus.valueOf(programStatus);
        String programRun = properties.get(ProgramOptionConstants.PROGRAM_RUN_ID);
        ProgramRunId programRunId = GSON.fromJson(programRun, ProgramRunId.class);
        // Should event publish happen for this status
        if (!shouldPublish(programRunId)) {
            return;
        }
        ProgramStatusEventDetails.Builder builder = ProgramStatusEventDetails.getBuilder(programRunId.getRun(), programRunId.getApplication(), programRunId.getProgram(), programRunId.getNamespace(), programStatus, RunIds.getTime(programRunId.getRun(), TimeUnit.MILLISECONDS));
        String userArgsString = properties.get(ProgramOptionConstants.USER_OVERRIDES);
        String sysArgsString = properties.get(ProgramOptionConstants.SYSTEM_OVERRIDES);
        Type argsMapType = new TypeToken<Map<String, String>>() {
        }.getType();
        builder = builder.withUserArgs(GSON.fromJson(userArgsString, argsMapType)).withSystemArgs(GSON.fromJson(sysArgsString, argsMapType));
        if (programRunStatus.isEndState()) {
            builder = populateErrorDetailsAndMetrics(builder, properties, programRunStatus, programRunId);
        }
        ProgramStatusEventDetails programStatusEventDetails = builder.build();
        ProgramStatusEvent programStatusEvent = new ProgramStatusEvent(publishTime, EVENT_VERSION, instanceName, projectName, programStatusEventDetails);
        programStatusEvents.add(programStatusEvent);
    });
    this.eventWriters.forEach(eventWriter -> eventWriter.write(programStatusEvents));
}