Examples with MDSKey io.cdap.cdap.data2.dataset2.lib.table.MDSKey used on opensource projects

Example 11 with MDSKey

use of io.cdap.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method getHistoricalRuns.

private Map<ProgramRunId, RunRecordMeta> getHistoricalRuns(@Nullable ProgramId programId, ProgramRunStatus status, final long startTime, final long endTime, int limit, @Nullable Predicate<RunRecordMeta> filter) {
    if (programId == null || !programId.getVersion().equals(ApplicationId.DEFAULT_VERSION)) {
        MDSKey key = getProgramKeyBuilder(TYPE_RUN_RECORD_COMPLETED, programId).build();
        return getHistoricalRuns(key, status, startTime, endTime, limit, null, filter);
    }
    Predicate<MDSKey> keyPredicate = new AppVersionPredicate(ApplicationId.DEFAULT_VERSION);
    MDSKey key = getProgramKeyBuilder(TYPE_RUN_RECORD_COMPLETED, programId).build();
    Map<ProgramRunId, RunRecordMeta> newRecords = getHistoricalRuns(key, status, startTime, endTime, limit, keyPredicate, filter);
    int remaining = limit - newRecords.size();
    if (remaining > 0 && !upgradeComplete.get()) {
        // We need to scan twice since the key is modified again in getHistoricalRuns since we want to use the
        // endTime and startTime to reduce the scan range
        key = getVersionLessProgramKeyBuilder(TYPE_RUN_RECORD_COMPLETED, programId).build();
        Map<ProgramRunId, RunRecordMeta> oldRecords = getHistoricalRuns(key, status, startTime, endTime, remaining, keyPredicate, filter);
        newRecords.putAll(oldRecords);
    }
    return newRecords;
}

Example 12 with MDSKey

use of io.cdap.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method getCompletedRun.

private RunRecordMeta getCompletedRun(ProgramRunId programRunId) {
    MDSKey completedKey = getProgramKeyBuilder(TYPE_RUN_RECORD_COMPLETED, programRunId.getParent()).build();
    RunRecordMeta runRecordMeta = getCompletedRun(completedKey, programRunId.getRun());
    if (!upgradeComplete.get() && runRecordMeta == null && programRunId.getVersion().equals(ApplicationId.DEFAULT_VERSION)) {
        completedKey = getVersionLessProgramKeyBuilder(TYPE_RUN_RECORD_COMPLETED, programRunId.getParent()).build();
        return getCompletedRun(completedKey, programRunId.getRun());
    }
    return runRecordMeta;
}

Example 13 with MDSKey

use of io.cdap.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method getWorkflowNodeStates.

/**
 * Return the {@link List} of {@link WorkflowNodeStateDetail} for a given Workflow run.
 */
public List<WorkflowNodeStateDetail> getWorkflowNodeStates(ProgramRunId workflowRunId) {
    MDSKey key = getProgramKeyBuilder(TYPE_WORKFLOW_NODE_STATE, workflowRunId).build();
    List<WorkflowNodeStateDetail> nodeStateDetails = list(key, WorkflowNodeStateDetail.class);
    // one specific run-id either in the old format or in the new format.
    if (!upgradeComplete.get() && nodeStateDetails.isEmpty() && workflowRunId.getVersion().equals(ApplicationId.DEFAULT_VERSION)) {
        key = getVersionLessProgramKeyBuilder(TYPE_WORKFLOW_NODE_STATE, workflowRunId).build();
        nodeStateDetails = list(key, WorkflowNodeStateDetail.class);
    }
    return nodeStateDetails;
}

Example 14 with MDSKey

use of io.cdap.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class WorkflowDataset method getNeighbors.

/**
 * Returns a map of WorkflowRunId to WorkflowRunRecord that are close to the WorkflowRunId provided by the user.
 *
 * @param id The workflow
 * @param runId The runid of the workflow
 * @param limit The limit on each side of the run that we want to see into
 * @param timeInterval The time interval that we want the results to be spaced apart
 * @return A Map of WorkflowRunId to the corresponding Workflow Run Record. A map is used so that duplicates of
 * the WorkflowRunRecord are not obtained
 */
private Map<String, WorkflowRunRecord> getNeighbors(WorkflowId id, RunId runId, int limit, long timeInterval) {
    long startTime = RunIds.getTime(runId, TimeUnit.SECONDS);
    Map<String, WorkflowRunRecord> workflowRunRecords = new HashMap<>();
    int i = -limit;
    long prevStartTime = startTime - (limit * timeInterval);
    // the last record was found if the (interval * the count of the loop) is less than the time.
    while (prevStartTime <= startTime + (limit * timeInterval)) {
        MDSKey mdsKey = getRowKeyBuilder(id, prevStartTime).build();
        byte[] startRowKey = mdsKey.getKey();
        Scan scan = new Scan(startRowKey, null);
        Scanner scanner = table.scan(scan);
        Row indexRow = scanner.next();
        if (indexRow == null) {
            return workflowRunRecords;
        }
        byte[] rowKey = indexRow.getRow();
        long time = ByteBuffer.wrap(rowKey, rowKey.length - Bytes.SIZEOF_LONG, Bytes.SIZEOF_LONG).getLong();
        if (!((time >= (startTime - (limit * timeInterval))) && time <= (startTime + (limit * timeInterval)))) {
            break;
        }
        Map<byte[], byte[]> columns = indexRow.getColumns();
        String workflowRunId = Bytes.toString(columns.get(RUNID));
        long timeTaken = Bytes.toLong(columns.get(TIME_TAKEN));
        List<ProgramRun> programRunList = GSON.fromJson(Bytes.toString(columns.get(NODES)), PROGRAM_RUNS_TYPE);
        workflowRunRecords.put(workflowRunId, new WorkflowRunRecord(workflowRunId, timeTaken, programRunList));
        prevStartTime = startTime + (i * timeInterval) < time ? time + 1 : startTime + (i * timeInterval);
        i++;
    }
    return workflowRunRecords;
}

Example 15 with MDSKey

use of io.cdap.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class WorkflowDataset method write.

void write(WorkflowId id, RunRecordMeta runRecordMeta, List<ProgramRun> programRunList) {
    long startTs = runRecordMeta.getStartTs();
    MDSKey mdsKey = getRowKeyBuilder(id, startTs).build();
    byte[] rowKey = mdsKey.getKey();
    Long stopTs = runRecordMeta.getStopTs();
    Preconditions.checkState(stopTs != null, "Workflow Stats are written when the workflow has completed. Hence, " + "expected workflow stop time to be non-null. Workflow = %s, Run = %s, Stop time = %s", id, runRecordMeta, stopTs);
    long timeTaken = stopTs - startTs;
    String value = GSON.toJson(programRunList, PROGRAM_RUNS_TYPE);
    table.put(rowKey, RUNID, Bytes.toBytes(runRecordMeta.getPid()));
    table.put(rowKey, TIME_TAKEN, Bytes.toBytes(timeTaken));
    table.put(rowKey, NODES, Bytes.toBytes(value));
}