Examples with MDSKey co.cask.cdap.data2.dataset2.lib.table.MDSKey used on opensource projects

Example 1 with MDSKey

use of co.cask.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class OrderedPair method getSecond.

public Set<SECOND> getSecond(Set<MDSKey> mdsKeys) {
    Set<SECOND> secondSet = Sets.newHashSetWithExpectedSize(mdsKeys.size());
    for (MDSKey mdsKey : mdsKeys) {
        MDSKey.Splitter splitter = mdsKey.split();
        // prefix
        splitter.skipString();
        keyMaker1.skipKey(splitter);
        secondSet.add(this.keyMaker2.getElement(splitter));
    }
    return secondSet;
}

Example 2 with MDSKey

use of co.cask.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class WorkflowDataset method getNeighbors.

/**
   * Returns a map of WorkflowRunId to WorkflowRunRecord that are close to the WorkflowRunId provided by the user.
   *
   * @param id The workflow
   * @param runId The runid of the workflow
   * @param limit The limit on each side of the run that we want to see into
   * @param timeInterval The time interval that we want the results to be spaced apart
   * @return A Map of WorkflowRunId to the corresponding Workflow Run Record. A map is used so that duplicates of
   * the WorkflowRunRecord are not obtained
   */
private Map<String, WorkflowRunRecord> getNeighbors(WorkflowId id, RunId runId, int limit, long timeInterval) {
    long startTime = RunIds.getTime(runId, TimeUnit.SECONDS);
    Map<String, WorkflowRunRecord> workflowRunRecords = new HashMap<>();
    int i = -limit;
    long prevStartTime = startTime - (limit * timeInterval);
    // the last record was found if the (interval * the count of the loop) is less than the time.
    while (prevStartTime <= startTime + (limit * timeInterval)) {
        MDSKey mdsKey = getRowKeyBuilder(id, prevStartTime).build();
        byte[] startRowKey = mdsKey.getKey();
        Scan scan = new Scan(startRowKey, null);
        Scanner scanner = table.scan(scan);
        Row indexRow = scanner.next();
        if (indexRow == null) {
            return workflowRunRecords;
        }
        byte[] rowKey = indexRow.getRow();
        long time = ByteBuffer.wrap(rowKey, rowKey.length - Bytes.SIZEOF_LONG, Bytes.SIZEOF_LONG).getLong();
        if (!((time >= (startTime - (limit * timeInterval))) && time <= (startTime + (limit * timeInterval)))) {
            break;
        }
        Map<byte[], byte[]> columns = indexRow.getColumns();
        String workflowRunId = Bytes.toString(columns.get(RUNID));
        long timeTaken = Bytes.toLong(columns.get(TIME_TAKEN));
        List<ProgramRun> programRunList = GSON.fromJson(Bytes.toString(columns.get(NODES)), PROGRAM_RUNS_TYPE);
        workflowRunRecords.put(workflowRunId, new WorkflowRunRecord(workflowRunId, timeTaken, programRunList));
        prevStartTime = startTime + (i * timeInterval) < time ? time + 1 : startTime + (i * timeInterval);
        i++;
    }
    return workflowRunRecords;
}

Example 3 with MDSKey

use of co.cask.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class DefaultPreviewStore method get.

@Override
public Map<String, List<JsonElement>> get(ApplicationId applicationId, String tracerName) {
    // PreviewStore is a singleton and we have to create gson for each operation since gson is not thread safe.
    Gson gson = new GsonBuilder().registerTypeAdapter(Schema.class, new SchemaTypeAdapter()).create();
    byte[] startRowKey = new MDSKey.Builder().add(applicationId.getNamespace()).add(applicationId.getApplication()).add(tracerName).build().getKey();
    byte[] stopRowKey = new MDSKey(Bytes.stopKeyForPrefix(startRowKey)).getKey();
    Map<String, List<JsonElement>> result = new HashMap<>();
    try (Scanner scanner = table.scan(startRowKey, stopRowKey, null, null, null)) {
        Row indexRow;
        while ((indexRow = scanner.next()) != null) {
            Map<byte[], byte[]> columns = indexRow.getColumns();
            String propertyName = Bytes.toString(columns.get(PROPERTY));
            JsonElement value = gson.fromJson(Bytes.toString(columns.get(VALUE)), JsonElement.class);
            List<JsonElement> values = result.get(propertyName);
            if (values == null) {
                values = new ArrayList<>();
                result.put(propertyName, values);
            }
            values.add(value);
        }
    } catch (IOException e) {
        String message = String.format("Error while reading preview data for application '%s' and tracer '%s'.", applicationId, tracerName);
        throw new RuntimeException(message, e);
    }
    return result;
}

Example 4 with MDSKey

use of co.cask.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method getHistoricalRuns.

private Map<ProgramRunId, RunRecordMeta> getHistoricalRuns(@Nullable ProgramId programId, ProgramRunStatus status, final long startTime, final long endTime, int limit, @Nullable Predicate<RunRecordMeta> filter) {
    if (programId == null || !programId.getVersion().equals(ApplicationId.DEFAULT_VERSION)) {
        MDSKey key = getProgramKeyBuilder(TYPE_RUN_RECORD_COMPLETED, programId).build();
        return getHistoricalRuns(key, status, startTime, endTime, limit, null, filter);
    }
    Predicate<MDSKey> keyPredicate = new AppVersionPredicate(ApplicationId.DEFAULT_VERSION);
    MDSKey key = getProgramKeyBuilder(TYPE_RUN_RECORD_COMPLETED, programId).build();
    Map<ProgramRunId, RunRecordMeta> newRecords = getHistoricalRuns(key, status, startTime, endTime, limit, keyPredicate, filter);
    int remaining = limit - newRecords.size();
    if (remaining > 0 && !upgradeComplete.get()) {
        // We need to scan twice since the key is modified again in getHistoricalRuns since we want to use the
        // endTime and startTime to reduce the scan range
        key = getVersionLessProgramKeyBuilder(TYPE_RUN_RECORD_COMPLETED, programId).build();
        Map<ProgramRunId, RunRecordMeta> oldRecords = getHistoricalRuns(key, status, startTime, endTime, remaining, keyPredicate, filter);
        newRecords.putAll(oldRecords);
    }
    return newRecords;
}

Example 5 with MDSKey

use of co.cask.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method getWorkflowNodeStates.

/**
   * Return the {@link List} of {@link WorkflowNodeStateDetail} for a given Workflow run.
   */
public List<WorkflowNodeStateDetail> getWorkflowNodeStates(ProgramRunId workflowRunId) {
    MDSKey key = getProgramKeyBuilder(TYPE_WORKFLOW_NODE_STATE, workflowRunId.getParent()).add(workflowRunId.getRun()).build();
    List<WorkflowNodeStateDetail> nodeStateDetails = list(key, WorkflowNodeStateDetail.class);
    // one specific run-id either in the old format or in the new format.
    if (!upgradeComplete.get() && nodeStateDetails.isEmpty() && workflowRunId.getVersion().equals(ApplicationId.DEFAULT_VERSION)) {
        key = getVersionLessProgramKeyBuilder(TYPE_WORKFLOW_NODE_STATE, workflowRunId.getParent()).add(workflowRunId.getRun()).build();
        nodeStateDetails = list(key, WorkflowNodeStateDetail.class);
    }
    return nodeStateDetails;
}