Examples with MDSKey co.cask.cdap.data2.dataset2.lib.table.MDSKey used on opensource projects

Example 21 with MDSKey

use of co.cask.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method getRunsForRunIds.

private Map<ProgramRunId, RunRecordMeta> getRunsForRunIds(final Set<ProgramRunId> runIds, String recordType, int limit) {
    Set<MDSKey> keySet = new HashSet<>();
    boolean includeVersionLessKeys = !upgradeComplete.get();
    for (ProgramRunId programRunId : runIds) {
        keySet.add(getProgramKeyBuilder(recordType, programRunId).build());
        if (includeVersionLessKeys && programRunId.getVersion().equals(ApplicationId.DEFAULT_VERSION)) {
            keySet.add(getVersionLessProgramKeyBuilder(recordType, programRunId).build());
        }
    }
    Predicate<KeyValue<RunRecordMeta>> combinedFilter = new Predicate<KeyValue<RunRecordMeta>>() {

        @Override
        public boolean apply(KeyValue<RunRecordMeta> input) {
            ProgramId programId = getProgramID(input.getKey());
            RunRecordMeta meta = input.getValue();
            ProgramRunId programRunId = programId.run(meta.getPid());
            return runIds.contains(programRunId);
        }
    };
    Map<MDSKey, RunRecordMeta> returnMap = listKV(keySet, RunRecordMeta.class, limit, combinedFilter);
    return getProgramRunIdMap(returnMap);
}

Example 22 with MDSKey

use of co.cask.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method writeApplication.

public void writeApplication(String namespaceId, String appId, String versionId, ApplicationSpecification spec) {
    if (!upgradeComplete.get() && versionId.equals(ApplicationId.DEFAULT_VERSION)) {
        MDSKey mdsKey = new MDSKey.Builder().add(TYPE_APP_META, namespaceId, appId).build();
        ApplicationMeta appMeta = get(mdsKey, ApplicationMeta.class);
        // If app meta exists for the application without a version, delete that key.
        if (appMeta != null) {
            delete(mdsKey);
        }
    }
    write(new MDSKey.Builder().add(TYPE_APP_META, namespaceId, appId, versionId).build(), new ApplicationMeta(appId, spec));
}

Example 23 with MDSKey

use of co.cask.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method addWorkflowNodeState.

private void addWorkflowNodeState(ProgramId programId, String pid, Map<String, String> systemArgs, ProgramRunStatus status, @Nullable BasicThrowable failureCause) {
    String workflowNodeId = systemArgs.get(ProgramOptionConstants.WORKFLOW_NODE_ID);
    String workflowName = systemArgs.get(ProgramOptionConstants.WORKFLOW_NAME);
    String workflowRun = systemArgs.get(ProgramOptionConstants.WORKFLOW_RUN_ID);
    ApplicationId appId = Ids.namespace(programId.getNamespace()).app(programId.getApplication());
    ProgramRunId workflowRunId = appId.workflow(workflowName).run(workflowRun);
    // Node states will be stored with following key:
    // workflowNodeState.namespace.app.WORKFLOW.workflowName.workflowRun.workflowNodeId
    MDSKey key = getProgramKeyBuilder(TYPE_WORKFLOW_NODE_STATE, workflowRunId.getParent()).add(workflowRun).add(workflowNodeId).build();
    WorkflowNodeStateDetail nodeStateDetail = new WorkflowNodeStateDetail(workflowNodeId, ProgramRunStatus.toNodeStatus(status), pid, failureCause);
    write(key, nodeStateDetail);
    // Get the run record of the Workflow which started this program
    key = getProgramKeyBuilder(TYPE_RUN_RECORD_STARTED, workflowRunId.getParent()).add(workflowRunId.getRun()).build();
    RunRecordMeta record = get(key, RunRecordMeta.class);
    if (record != null) {
        // Update the parent Workflow run record by adding node id and program run id in the properties
        Map<String, String> properties = record.getProperties();
        properties.put(workflowNodeId, pid);
        write(key, new RunRecordMeta(record, properties));
    }
}

Example 24 with MDSKey

use of co.cask.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method deleteApplication.

public void deleteApplication(String namespaceId, String appId, String versionId) {
    if (!upgradeComplete.get() && versionId.equals(ApplicationId.DEFAULT_VERSION)) {
        MDSKey mdsKey = new MDSKey.Builder().add(TYPE_APP_META, namespaceId, appId).build();
        ApplicationMeta appMeta = get(mdsKey, ApplicationMeta.class);
        // If app meta exists for the application without a version, delete only that key.
        if (appMeta != null) {
            delete(mdsKey);
        }
    }
    deleteAll(new MDSKey.Builder().add(TYPE_APP_META, namespaceId, appId, versionId).build());
}

Example 25 with MDSKey

use of co.cask.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method getRunningInRangeForStatus.

@VisibleForTesting
List<Iterable<RunId>> getRunningInRangeForStatus(String statusKey, final long startTimeInSecs, final long endTimeInSecs, long maxScanTimeMillis, Ticker ticker) {
    // Create time filter to get running programs between start and end time
    Predicate<RunRecordMeta> timeFilter = new Predicate<RunRecordMeta>() {

        @Override
        public boolean apply(RunRecordMeta runRecordMeta) {
            // or program's stop time was after startTime
            return runRecordMeta.getStartTs() < endTimeInSecs && (runRecordMeta.getStopTs() == null || runRecordMeta.getStopTs() >= startTimeInSecs);
        }
    };
    // Break up scans into smaller batches to prevent transaction timeout
    List<Iterable<RunId>> batches = new ArrayList<>();
    MDSKey startKey = new MDSKey.Builder().add(statusKey).build();
    MDSKey endKey = new MDSKey(Bytes.stopKeyForPrefix(startKey.getKey()));
    while (true) {
        ScanFunction scanFunction = new ScanFunction(timeFilter, ticker, maxScanTimeMillis);
        scanFunction.start();
        scan(startKey, endKey, RunRecordMeta.class, scanFunction);
        // stop when scan returns zero elements
        if (scanFunction.getNumProcessed() == 0) {
            break;
        }
        batches.add(Iterables.transform(scanFunction.getValues(), RUN_RECORD_META_TO_RUN_ID_FUNCTION));
        // key for next scan is the last key + 1 from the previous scan
        startKey = new MDSKey(Bytes.stopKeyForPrefix(scanFunction.getLastKey().getKey()));
    }
    return batches;
}