Examples with MDSKey io.cdap.cdap.data2.dataset2.lib.table.MDSKey used on opensource projects

Example 71 with MDSKey

use of io.cdap.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method getActiveRuns.

public Map<ProgramRunId, RunRecordMeta> getActiveRuns(ApplicationId applicationId) {
    Predicate<RunRecordMeta> timePredicate = getTimeRangePredicate(0, Long.MAX_VALUE);
    MDSKey key = getApplicationKeyBuilder(TYPE_RUN_RECORD_STARTING, applicationId).build();
    Map<ProgramRunId, RunRecordMeta> activeRuns = getProgramRunIdMap(listKV(key, null, RunRecordMeta.class, Integer.MAX_VALUE, timePredicate));
    key = getApplicationKeyBuilder(TYPE_RUN_RECORD_STARTED, applicationId).build();
    activeRuns.putAll(getProgramRunIdMap(listKV(key, null, RunRecordMeta.class, Integer.MAX_VALUE, timePredicate)));
    key = getApplicationKeyBuilder(TYPE_RUN_RECORD_SUSPENDED, applicationId).build();
    activeRuns.putAll(getProgramRunIdMap(listKV(key, null, RunRecordMeta.class, Integer.MAX_VALUE, timePredicate)));
    return activeRuns;
}

Example 72 with MDSKey

use of io.cdap.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method getProgramRunIdMap.

/**
 * Converts MDSkeys in the map to ProgramIDs
 *
 * @param keymap map with keys as MDSkeys
 * @return map with keys as program IDs
 */
private Map<ProgramRunId, RunRecordMeta> getProgramRunIdMap(Map<MDSKey, RunRecordMeta> keymap) {
    Map<ProgramRunId, RunRecordMeta> programRunIdMap = new LinkedHashMap<>();
    for (Map.Entry<MDSKey, RunRecordMeta> entry : keymap.entrySet()) {
        ProgramId programId = getProgramID(entry.getKey());
        programRunIdMap.put(programId.run(entry.getValue().getPid()), entry.getValue());
    }
    return programRunIdMap;
}

Example 73 with MDSKey

use of io.cdap.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method recordProgramProvisioning.

/**
 * Record that the program run is provisioning compute resources for the run. If the current status has
 * a higher source id, this call will be ignored.
 *
 * @param programRunId program run
 * @param startTs start time of the run
 * @param runtimeArgs runtime arguments
 * @param systemArgs system arguments
 * @param sourceId unique id representing the source of program run status, such as the message id of the program
 *                 run status notification in TMS. The source id must increase as the recording time of the program
 *                 run status increases, so that the attempt to persist program run status older than the existing
 *                 program run status will be ignored
 * @return {@link ProgramRunClusterStatus#PROVISIONING} if it is successfully persisted, {@code null} otherwise.
 */
@Nullable
public ProgramRunClusterStatus recordProgramProvisioning(ProgramRunId programRunId, long startTs, Map<String, String> runtimeArgs, Map<String, String> systemArgs, byte[] sourceId) {
    MDSKey key = getProgramKeyBuilder(TYPE_RUN_RECORD_STARTING, programRunId).build();
    RunRecordMeta existing = getRun(programRunId);
    // for some reason, there is an existing run record.
    if (existing != null) {
        // runtime args, and system args of the existing run record.
        if (!isValid(existing, sourceId, "provisioning")) {
            return null;
        }
        return null;
    }
    String workflowRunId = null;
    if (systemArgs != null && systemArgs.containsKey(ProgramOptionConstants.WORKFLOW_NAME)) {
        workflowRunId = systemArgs.get(ProgramOptionConstants.WORKFLOW_RUN_ID);
    }
    ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
    builder.put("runtimeArgs", GSON.toJson(runtimeArgs, MAP_STRING_STRING_TYPE));
    if (workflowRunId != null) {
        builder.put("workflowrunid", workflowRunId);
    }
    ProgramRunCluster cluster = new ProgramRunCluster(ProgramRunClusterStatus.PROVISIONING, null, null);
    RunRecordMeta meta = RunRecordMeta.builder().setProgramRunId(programRunId).setStartTime(startTs).setStatus(ProgramRunStatus.STARTING).setProperties(builder.build()).setSystemArgs(systemArgs).setCluster(cluster).setSourceId(sourceId).build();
    write(key, meta);
    return ProgramRunClusterStatus.PROVISIONING;
}

Example 74 with MDSKey

use of io.cdap.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method recordProgramRunningOldFormat.

/**
 * Logs start of program run and persists program status to {@link ProgramRunStatus#STARTING} with version-less key.
 * See {@link #recordProgramRunning(ProgramRunId, long, String, byte[])}
 */
@VisibleForTesting
void recordProgramRunningOldFormat(ProgramRunId programRunId, long stateChangeTime, String twillRunId, byte[] sourceId) {
    MDSKey key = getVersionLessProgramKeyBuilder(TYPE_RUN_RECORD_STARTED, programRunId).build();
    recordProgramRunning(programRunId, stateChangeTime, twillRunId, key, sourceId);
}

Example 75 with MDSKey

use of io.cdap.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method recordProgramStop.

private ProgramRunStatus recordProgramStop(ProgramRunId programRunId, long stopTs, ProgramRunStatus runStatus, @Nullable BasicThrowable failureCause, MDSKey.Builder builder, byte[] sourceId) {
    RunRecordMeta existing = getRun(programRunId);
    if (existing == null) {
        LOG.warn("Ignoring unexpected transition of program run {} to program state {} with no existing run record.", programRunId, runStatus);
        return null;
    }
    if (!isValid(existing, sourceId, runStatus.name().toLowerCase())) {
        // Skip recording stop if the existing records are not valid
        return null;
    }
    if (existing.getStatus().isEndState()) {
        LOG.warn("Ignoring unexpected transition of program run {} from program state {} to program state {}.", programRunId, existing.getStatus(), runStatus);
        return null;
    }
    // Delete the old run record
    delete(existing);
    // Record in the workflow
    Map<String, String> systemArgs = existing.getSystemArgs();
    if (systemArgs != null && systemArgs.containsKey(ProgramOptionConstants.WORKFLOW_NAME)) {
        addWorkflowNodeState(programRunId, systemArgs, runStatus, failureCause, sourceId);
    }
    MDSKey key = builder.add(getInvertedTsKeyPart(existing.getStartTs())).add(programRunId.getRun()).build();
    write(key, RunRecordMeta.builder(existing).setStopTime(stopTs).setStatus(runStatus).setSourceId(sourceId).build());
    return runStatus;
}