Examples with ProgramHeartbeatTable io.cdap.cdap.reporting.ProgramHeartbeatTable used on opensource projects

Example 1 with ProgramHeartbeatTable

use of io.cdap.cdap.reporting.ProgramHeartbeatTable in project cdap by caskdata.

the class ProgramNotificationSubscriberService method processMessages.

@Override
protected void processMessages(StructuredTableContext structuredTableContext, Iterator<ImmutablePair<String, Notification>> messages) throws Exception {
    ProgramHeartbeatTable heartbeatDataset = new ProgramHeartbeatTable(structuredTableContext);
    List<Runnable> tasks = new LinkedList<>();
    while (messages.hasNext()) {
        ImmutablePair<String, Notification> messagePair = messages.next();
        List<Runnable> runnables = processNotification(heartbeatDataset, messagePair.getFirst().getBytes(StandardCharsets.UTF_8), messagePair.getSecond(), structuredTableContext);
        tasks.addAll(runnables);
    }
    // Only add post processing tasks if all messages are processed. If there is exception in the processNotifiation,
    // messages will be replayed.
    this.tasks.addAll(tasks);
}

Example 2 with ProgramHeartbeatTable

use of io.cdap.cdap.reporting.ProgramHeartbeatTable in project cdap by caskdata.

the class ProgramNotificationSubscriberService method processWorkflowOnStop.

/**
 * On workflow program stop, inspects inner program states and adjust them if they are not in end state already.
 *
 * @param appMetadataStore the {@link AppMetadataStore} to write the status to
 * @param programHeartbeatTable the {@link ProgramHeartbeatTable} to write the status to
 * @param programRunId the program run of the completed program
 * @param programRunStatus the status of the completion
 * @param notification the {@link Notification} that carries information about the workflow completion
 * @param sourceId the source message id of the notification
 * @param runnables a {@link List} adding {@link Runnable} to be executed after event handling is completed
 * @throws Exception if failed to update program status
 */
private void processWorkflowOnStop(AppMetadataStore appMetadataStore, ProgramHeartbeatTable programHeartbeatTable, ProgramRunId programRunId, ProgramRunStatus programRunStatus, Notification notification, byte[] sourceId, List<Runnable> runnables) throws Exception {
    ApplicationId appId = programRunId.getParent().getParent();
    WorkflowSpecification workflowSpec = Optional.ofNullable(appMetadataStore.getApplication(appId)).map(appMeta -> appMeta.getSpec().getWorkflows().get(programRunId.getProgram())).orElse(null);
    // If cannot find the workflow spec (e.g. app deleted), then there is nothing we can do.
    if (workflowSpec == null) {
        return;
    }
    // For all MR and Spark nodes, we need to update the inner program run status if they are not in end state yet.
    for (WorkflowNode workflowNode : workflowSpec.getNodeIdMap().values()) {
        if (!(workflowNode instanceof WorkflowActionNode)) {
            continue;
        }
        ScheduleProgramInfo programInfo = ((WorkflowActionNode) workflowNode).getProgram();
        if (!WORKFLOW_INNER_PROGRAM_TYPES.containsKey(programInfo.getProgramType())) {
            continue;
        }
        // Get all active runs of the inner program. If the parent workflow runId is the same as this one,
        // set a terminal state for the inner program run.
        ProgramId innerProgramId = appId.program(WORKFLOW_INNER_PROGRAM_TYPES.get(programInfo.getProgramType()), programInfo.getProgramName());
        Map<ProgramRunId, Notification> innerProgramNotifications = new LinkedHashMap<>();
        appMetadataStore.scanActiveRuns(innerProgramId, runRecord -> {
            Map<String, String> systemArgs = runRecord.getSystemArgs();
            String workflowName = systemArgs.get(ProgramOptionConstants.WORKFLOW_NAME);
            String workflowRun = systemArgs.get(ProgramOptionConstants.WORKFLOW_RUN_ID);
            if (workflowName == null || workflowRun == null) {
                return;
            }
            ProgramRunId workflowRunId = appId.program(ProgramType.WORKFLOW, workflowName).run(workflowRun);
            if (!programRunId.equals(workflowRunId)) {
                return;
            }
            Map<String, String> notificationProps = new HashMap<>(notification.getProperties());
            notificationProps.put(ProgramOptionConstants.PROGRAM_RUN_ID, GSON.toJson(runRecord.getProgramRunId()));
            innerProgramNotifications.put(runRecord.getProgramRunId(), new Notification(Notification.Type.PROGRAM_STATUS, notificationProps));
        });
        for (Map.Entry<ProgramRunId, Notification> entry : innerProgramNotifications.entrySet()) {
            handleProgramEvent(entry.getKey(), programRunStatus, entry.getValue(), sourceId, appMetadataStore, programHeartbeatTable, runnables);
        }
    }
}

Example 3 with ProgramHeartbeatTable

use of io.cdap.cdap.reporting.ProgramHeartbeatTable in project cdap by caskdata.

the class ProgramNotificationSubscriberServiceTest method cleanupTest.

@After
public void cleanupTest() {
    TransactionRunners.run(transactionRunner, context -> {
        new ProgramHeartbeatTable(context).deleteAll();
        AppMetadataStore.create(context).deleteAllAppMetadataTables();
    });
}