Examples with ProgramRunStatus io.cdap.cdap.proto.ProgramRunStatus used on opensource projects

Example 11 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by cdapio.

the class ProgramNotificationSubscriberService method processNotification.

/**
 * Process a {@link Notification} received from TMS.
 *
 * @param programHeartbeatTable the {@link ProgramHeartbeatTable} for writing heart beats and program status
 * @param messageIdBytes the raw message id in the TMS for the notification
 * @param notification the {@link Notification} to process
 * @param context context to get the table for operations
 * @return a {@link List} of {@link Runnable} tasks to run after the transactional processing of the whole
 *         messages batch is completed
 * @throws Exception if failed to process the given notification
 */
private List<Runnable> processNotification(ProgramHeartbeatTable programHeartbeatTable, byte[] messageIdBytes, Notification notification, StructuredTableContext context) throws Exception {
    AppMetadataStore appMetadataStore = AppMetadataStore.create(context);
    Map<String, String> properties = notification.getProperties();
    // Required parameters
    String programRun = properties.get(ProgramOptionConstants.PROGRAM_RUN_ID);
    String programStatusStr = properties.get(ProgramOptionConstants.PROGRAM_STATUS);
    String clusterStatusStr = properties.get(ProgramOptionConstants.CLUSTER_STATUS);
    // Ignore notifications which specify an invalid ProgramRunId, which shouldn't happen
    if (programRun == null) {
        LOG.warn("Ignore notification that misses program run state information, {}", notification);
        return Collections.emptyList();
    }
    ProgramRunId programRunId = GSON.fromJson(programRun, ProgramRunId.class);
    ProgramRunStatus programRunStatus = null;
    if (programStatusStr != null) {
        try {
            programRunStatus = ProgramRunStatus.valueOf(programStatusStr);
        } catch (IllegalArgumentException e) {
            LOG.warn("Ignore notification with invalid program run status {} for program {}, {}", programStatusStr, programRun, notification);
            return Collections.emptyList();
        }
    }
    ProgramRunClusterStatus clusterStatus = null;
    if (clusterStatusStr != null) {
        try {
            clusterStatus = ProgramRunClusterStatus.valueOf(clusterStatusStr);
        } catch (IllegalArgumentException e) {
            LOG.warn("Ignore notification with invalid program run cluster status {} for program {}", clusterStatusStr, programRun);
            return Collections.emptyList();
        }
    }
    if (notification.getNotificationType().equals(Notification.Type.PROGRAM_HEART_BEAT)) {
        RunRecordDetail runRecordMeta = appMetadataStore.getRun(programRunId);
        long heartBeatTimeInSeconds = TimeUnit.MILLISECONDS.toSeconds(Long.parseLong(properties.get(ProgramOptionConstants.HEART_BEAT_TIME)));
        writeToHeartBeatTable(runRecordMeta, heartBeatTimeInSeconds, programHeartbeatTable);
        // we can return after writing to heart beat table
        return Collections.emptyList();
    }
    List<Runnable> result = new ArrayList<>();
    if (programRunStatus != null) {
        handleProgramEvent(programRunId, programRunStatus, notification, messageIdBytes, appMetadataStore, programHeartbeatTable, result);
    }
    if (clusterStatus == null) {
        return result;
    }
    handleClusterEvent(programRunId, clusterStatus, notification, messageIdBytes, appMetadataStore, context).ifPresent(result::add);
    return result;
}

Example 12 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by cdapio.

the class AppMetadataStore method recordProgramSuspendResume.

private RunRecordDetail recordProgramSuspendResume(ProgramRunId programRunId, byte[] sourceId, RunRecordDetail existing, String action, long timestamp) throws IOException {
    ProgramRunStatus toStatus = ProgramRunStatus.SUSPENDED;
    if (action.equals("resume")) {
        toStatus = ProgramRunStatus.RUNNING;
    }
    // Delete the old run record
    delete(existing);
    List<Field<?>> key = getProgramRunInvertedTimeKey(TYPE_RUN_RECORD_ACTIVE, programRunId, existing.getStartTs());
    RunRecordDetail.Builder builder = RunRecordDetail.builder(existing).setStatus(toStatus).setSourceId(sourceId);
    if (timestamp != -1) {
        if (action.equals("resume")) {
            builder.setResumeTime(timestamp);
        } else {
            builder.setSuspendTime(timestamp);
        }
    }
    RunRecordDetail meta = builder.build();
    writeToRunRecordTableWithPrimaryKeys(key, meta);
    LOG.trace("Recorded {} for program {}", toStatus, programRunId);
    return meta;
}

Example 13 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by cdapio.

the class DefaultStore method getRuns.

@Override
public List<ProgramHistory> getRuns(Collection<ProgramId> programs, ProgramRunStatus status, long startTime, long endTime, int limitPerProgram) {
    return TransactionRunners.run(transactionRunner, context -> {
        List<ProgramHistory> result = new ArrayList<>(programs.size());
        AppMetadataStore appMetadataStore = getAppMetadataStore(context);
        Set<ProgramId> existingPrograms = appMetadataStore.filterProgramsExistence(programs);
        for (ProgramId programId : programs) {
            if (!existingPrograms.contains(programId)) {
                result.add(new ProgramHistory(programId, Collections.emptyList(), new ProgramNotFoundException(programId)));
                continue;
            }
            List<RunRecord> runs = appMetadataStore.getRuns(programId, status, startTime, endTime, limitPerProgram, null).values().stream().map(record -> RunRecord.builder(record).build()).collect(Collectors.toList());
            result.add(new ProgramHistory(programId, runs, null));
        }
        return result;
    });
}

Example 14 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by cdapio.

the class RuntimeClientServiceTest method testProgramTerminate.

/**
 * Test for {@link RuntimeClientService} that will terminate itself when seeing program completed message.
 */
@Test
public void testProgramTerminate() throws Exception {
    MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
    MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
    ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
    // Send a terminate program state first, wait for the service sees the state change,
    // then publish messages to other topics.
    programStateWriter.completed(PROGRAM_RUN_ID);
    Tasks.waitFor(true, () -> runtimeClientService.getProgramFinishTime() >= 0, 2, TimeUnit.SECONDS);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        // the RuntimeClientService will decode it to watch for program termination
        if (!entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            List<String> payloads = Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey());
            messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), StandardCharsets.UTF_8, payloads.iterator());
        }
    }
    // The client service should get stopped by itself.
    Tasks.waitFor(Service.State.TERMINATED, () -> runtimeClientService.state(), clientCConf.getLong(Constants.RuntimeMonitor.GRACEFUL_SHUTDOWN_MS) + 2000, TimeUnit.MILLISECONDS);
    // All messages should be sent after the runtime client service stopped
    MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            // Extract the program run status from the Notification
            Tasks.waitFor(Collections.singletonList(ProgramRunStatus.COMPLETED), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        } else {
            Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        }
    }
}

Example 15 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by caskdata.

the class RuntimeProgramStatusSubscriberService method processNotification.

/**
 * Processes a given {@link Notification} from TMS and updates the {@link AppMetadataStore}.
 *
 * @param sourceId the message id in TMS
 * @param notification the {@link Notification} to process
 * @param store the {@link AppMetadataStore} to write to
 * @throws IOException if failed to write to the store
 */
private void processNotification(byte[] sourceId, Notification notification, AppMetadataStore store) throws IOException {
    Map<String, String> properties = notification.getProperties();
    ProgramRunId programRunId;
    ProgramRunStatus programRunStatus;
    try {
        programRunId = GSON.fromJson(properties.get(ProgramOptionConstants.PROGRAM_RUN_ID), ProgramRunId.class);
        if (programRunId == null) {
            throw new IllegalArgumentException("Missing program run id from notification");
        }
        programRunStatus = ProgramRunStatus.valueOf(properties.get(ProgramOptionConstants.PROGRAM_STATUS));
    } catch (Exception e) {
        // This shouldn't happen. If it does, we can only log and ignore the event.
        LOG.warn("Ignore notification due to unable to get program run id and program run status from notification {}", notification, e);
        return;
    }
    // Runtime server only needs the programRunId and status. We use the AppMetadataStore to record to help
    // handling state transition correctly, but we omit certain fields when calling those record* methods since
    // they are not used by the runtime server.
    LOG.debug("Received program {} of status {} {}", programRunId, programRunStatus, Bytes.toString(sourceId));
    switch(programRunStatus) {
        case STARTING:
            {
                ProgramOptions programOptions = ProgramOptions.fromNotification(notification, GSON);
                // Strip off user args and trim down system args as runtime only needs the run status for validation purpose.
                // User and system args could be large and store them in local store can lead to unnecessary storage
                // and processing overhead.
                store.recordProgramProvisioning(programRunId, Collections.emptyMap(), RuntimeMonitors.trimSystemArgs(programOptions.getArguments().asMap()), sourceId, null);
                store.recordProgramProvisioned(programRunId, 0, sourceId);
                store.recordProgramStart(programRunId, null, programOptions.getArguments().asMap(), sourceId);
                break;
            }
        case RUNNING:
            store.recordProgramRunning(programRunId, Optional.ofNullable(properties.get(ProgramOptionConstants.LOGICAL_START_TIME)).map(Long::parseLong).orElse(System.currentTimeMillis()), null, sourceId);
            break;
        case SUSPENDED:
            store.recordProgramSuspend(programRunId, sourceId, Optional.ofNullable(properties.get(ProgramOptionConstants.SUSPEND_TIME)).map(Long::parseLong).orElse(System.currentTimeMillis()));
            break;
        case STOPPING:
            store.recordProgramStopping(programRunId, sourceId, Optional.ofNullable(properties.get(ProgramOptionConstants.STOPPING_TIME)).map(Long::parseLong).orElse(System.currentTimeMillis()), Optional.ofNullable(properties.get(ProgramOptionConstants.TERMINATE_TIME)).map(Long::parseLong).orElse(Long.MAX_VALUE));
            break;
        case RESUMING:
            store.recordProgramResumed(programRunId, sourceId, Optional.ofNullable(properties.get(ProgramOptionConstants.RESUME_TIME)).map(Long::parseLong).orElse(System.currentTimeMillis()));
            break;
        case COMPLETED:
        case FAILED:
        case KILLED:
            store.recordProgramStop(programRunId, Optional.ofNullable(properties.get(ProgramOptionConstants.END_TIME)).map(Long::parseLong).orElse(System.currentTimeMillis()), programRunStatus, null, sourceId);
            // We don't need to retain records for terminated programs, hence just delete it
            store.deleteRunIfTerminated(programRunId, sourceId);
            break;
        case REJECTED:
            {
                ProgramDescriptor programDescriptor = GSON.fromJson(properties.get(ProgramOptionConstants.PROGRAM_DESCRIPTOR), ProgramDescriptor.class);
                // Strip off user args and trim down system args as runtime only needs the run status for validation purpose.
                // User and system args could be large and store them in local store can lead to unnecessary storage
                // and processing overhead.
                store.recordProgramRejected(programRunId, Collections.emptyMap(), Collections.emptyMap(), sourceId, programDescriptor.getArtifactId().toApiArtifactId());
                // We don't need to retain records for terminated programs, hence just delete it
                store.deleteRunIfTerminated(programRunId, sourceId);
                break;
            }
    }
}