Examples with MDSKey co.cask.cdap.data2.dataset2.lib.table.MDSKey used on opensource projects

Example 26 with MDSKey

use of co.cask.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method getAllAppVersionsAppIds.

public List<ApplicationId> getAllAppVersionsAppIds(String namespaceId, String appId) {
    List<ApplicationId> appIds = new ArrayList<>();
    for (MDSKey key : listKV(new MDSKey.Builder().add(TYPE_APP_META, namespaceId, appId).build(), ApplicationMeta.class).keySet()) {
        MDSKey.Splitter splitter = key.split();
        // skip recordType
        splitter.skipBytes();
        // skip namespaceId
        splitter.skipBytes();
        // skip appId
        splitter.skipBytes();
        String versionId = splitter.hasRemaining() ? splitter.getString() : ApplicationId.DEFAULT_VERSION;
        appIds.add(new NamespaceId(namespaceId).app(appId, versionId));
    }
    return appIds;
}

Example 27 with MDSKey

use of co.cask.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method getNonCompleteRuns.

private Map<ProgramRunId, RunRecordMeta> getNonCompleteRuns(@Nullable ProgramId programId, String recordType, final long startTime, final long endTime, int limit, Predicate<RunRecordMeta> filter) {
    Predicate<RunRecordMeta> valuePredicate = andPredicate(new Predicate<RunRecordMeta>() {

        @Override
        public boolean apply(RunRecordMeta input) {
            return input.getStartTs() >= startTime && input.getStartTs() < endTime;
        }
    }, filter);
    if (programId == null || !programId.getVersion().equals(ApplicationId.DEFAULT_VERSION)) {
        MDSKey key = getProgramKeyBuilder(recordType, programId).build();
        return getProgramRunIdMap(listKV(key, null, RunRecordMeta.class, limit, valuePredicate));
    }
    Predicate<MDSKey> keyPredicate = new AppVersionPredicate(ApplicationId.DEFAULT_VERSION);
    MDSKey key = getProgramKeyBuilder(recordType, programId).build();
    Map<MDSKey, RunRecordMeta> newRecords = listKV(key, null, RunRecordMeta.class, limit, keyPredicate, valuePredicate);
    int remaining = limit - newRecords.size();
    if (remaining > 0 && !upgradeComplete.get()) {
        // We need to scan twice since the scan key is modified based on whether we include the app version or not.
        key = getVersionLessProgramKeyBuilder(recordType, programId).build();
        Map<MDSKey, RunRecordMeta> oldRecords = listKV(key, null, RunRecordMeta.class, remaining, keyPredicate, valuePredicate);
        newRecords.putAll(oldRecords);
    }
    return getProgramRunIdMap(newRecords);
}

Example 28 with MDSKey

use of co.cask.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method recordProgramSuspendResume.

private void recordProgramSuspendResume(ProgramId programId, String pid, String action) {
    String fromType = TYPE_RUN_RECORD_STARTED;
    String toType = TYPE_RUN_RECORD_SUSPENDED;
    ProgramRunStatus toStatus = ProgramRunStatus.SUSPENDED;
    if (action.equals("resume")) {
        fromType = TYPE_RUN_RECORD_SUSPENDED;
        toType = TYPE_RUN_RECORD_STARTED;
        toStatus = ProgramRunStatus.RUNNING;
    }
    MDSKey key = getProgramKeyBuilder(fromType, programId).add(pid).build();
    RunRecordMeta record = get(key, RunRecordMeta.class);
    // Check without the version string only for default version
    if (!upgradeComplete.get() && record == null && (programId.getVersion().equals(ApplicationId.DEFAULT_VERSION))) {
        key = getVersionLessProgramKeyBuilder(fromType, programId).add(pid).build();
        record = get(key, RunRecordMeta.class);
    }
    if (record == null) {
        String msg = String.format("No meta for %s run record for namespace %s app %s program type %s " + "program %s pid %s exists", action.equals("suspend") ? "started" : "suspended", programId.getNamespace(), programId.getApplication(), programId.getType().name(), programId.getProgram(), pid);
        LOG.error(msg);
        throw new IllegalArgumentException(msg);
    }
    // Since the key contains the RunId/PID in addition to the programId, it is ok to deleteAll.
    deleteAll(key);
    key = getProgramKeyBuilder(toType, programId).add(pid).build();
    write(key, new RunRecordMeta(record, null, toStatus));
}

Example 29 with MDSKey

use of co.cask.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method upgradeVersionKeys.

/**
   * Upgrades the rowkeys for the given record type.
   *
   * @param recordType type of the record
   * @param typeOfT    content type of the record
   * @param <T>        type param
   * @param maxRows    maximum number of rows to be upgraded in this call.
   * @return true if no rows required an upgrade
   */
private <T> boolean upgradeVersionKeys(String recordType, Type typeOfT, int maxRows) {
    LOG.info("Upgrading {}", recordType);
    MDSKey startKey = new MDSKey.Builder().add(recordType).build();
    Map<MDSKey, T> oldMap = listKV(startKey, typeOfT);
    Map<MDSKey, T> newMap = new HashMap<>();
    Set<MDSKey> deleteKeys = new HashSet<>();
    for (Map.Entry<MDSKey, T> oldEntry : oldMap.entrySet()) {
        MDSKey oldKey = oldEntry.getKey();
        MDSKey newKey = appendDefaultVersion(recordType, typeOfT, oldKey);
        // If the key has been modified, only then add it to the map.
        if (!newKey.equals(oldKey)) {
            deleteKeys.add(oldKey);
            // If a row with the new key doesn't exists, only then upgrade the old key otherwise just delete the old key.
            Object valueOfNewKey = get(newKey, typeOfT);
            if (valueOfNewKey == null) {
                newMap.put(newKey, oldEntry.getValue());
            }
            // We want to modify only certain number of rows
            if (deleteKeys.size() >= maxRows) {
                break;
            }
        }
    }
    // No rows needs to be modified
    if (deleteKeys.size() == 0) {
        return true;
    }
    // Delete old keys
    for (MDSKey oldKey : deleteKeys) {
        delete(oldKey);
    }
    // Write new rows
    for (Map.Entry<MDSKey, T> newEntry : newMap.entrySet()) {
        write(newEntry.getKey(), newEntry.getValue());
    }
    return false;
}

Example 30 with MDSKey

use of co.cask.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method recordProgramStop.

private void recordProgramStop(ProgramId programId, String pid, long stopTs, ProgramRunStatus runStatus, @Nullable BasicThrowable failureCause, MDSKey.Builder builder) {
    MDSKey key = getProgramKeyBuilder(TYPE_RUN_RECORD_STARTED, programId).add(pid).build();
    RunRecordMeta started = getFirst(key, RunRecordMeta.class);
    // Check without the version string only for default version
    if (!upgradeComplete.get() && started == null && (programId.getVersion().equals(ApplicationId.DEFAULT_VERSION))) {
        key = getVersionLessProgramKeyBuilder(TYPE_RUN_RECORD_STARTED, programId).add(pid).build();
        started = getFirst(key, RunRecordMeta.class);
    }
    if (started == null) {
        String msg = String.format("No meta for started run record for namespace %s app %s version %s program type %s " + "program %s pid %s exists", programId.getNamespace(), programId.getApplication(), programId.getVersion(), programId.getType().name(), programId.getProgram(), pid);
        LOG.error(msg);
        throw new IllegalArgumentException(msg);
    }
    if (started.getSystemArgs() != null && started.getSystemArgs().containsKey(ProgramOptionConstants.WORKFLOW_NAME)) {
        addWorkflowNodeState(programId, pid, started.getSystemArgs(), runStatus, failureCause);
    }
    // Since the key contains the RunId/PID in addition to the programId, it is ok to deleteAll.
    deleteAll(key);
    key = builder.add(getInvertedTsKeyPart(started.getStartTs())).add(pid).build();
    write(key, new RunRecordMeta(started, stopTs, runStatus));
}