Examples with MDSKey io.cdap.cdap.data2.dataset2.lib.table.MDSKey used on opensource projects

Example 66 with MDSKey

use of io.cdap.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method getAllAppVersionsAppIds.

public List<ApplicationId> getAllAppVersionsAppIds(String namespaceId, String appId) {
    List<ApplicationId> appIds = new ArrayList<>();
    for (MDSKey key : listKV(new MDSKey.Builder().add(TYPE_APP_META, namespaceId, appId).build(), ApplicationMeta.class).keySet()) {
        MDSKey.Splitter splitter = key.split();
        // skip recordType
        splitter.skipBytes();
        // skip namespaceId
        splitter.skipBytes();
        // skip appId
        splitter.skipBytes();
        String versionId = splitter.hasRemaining() ? splitter.getString() : ApplicationId.DEFAULT_VERSION;
        appIds.add(new NamespaceId(namespaceId).app(appId, versionId));
    }
    return appIds;
}

Example 67 with MDSKey

use of io.cdap.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method getRunningInRangeForStatus.

@VisibleForTesting
List<Iterable<RunId>> getRunningInRangeForStatus(String statusKey, final long startTimeInSecs, final long endTimeInSecs, long maxScanTimeMillis, Ticker ticker) {
    // Create time filter to get running programs between start and end time
    Predicate<RunRecordMeta> timeFilter = (runRecordMeta) -> runRecordMeta.getStartTs() < endTimeInSecs && (runRecordMeta.getStopTs() == null || runRecordMeta.getStopTs() >= startTimeInSecs);
    // Break up scans into smaller batches to prevent transaction timeout
    List<Iterable<RunId>> batches = new ArrayList<>();
    MDSKey startKey = new MDSKey.Builder().add(statusKey).build();
    MDSKey endKey = new MDSKey(Bytes.stopKeyForPrefix(startKey.getKey()));
    while (true) {
        ScanFunction scanFunction = new ScanFunction(timeFilter, ticker, maxScanTimeMillis);
        scanFunction.start();
        scan(startKey, endKey, RunRecordMeta.class, scanFunction);
        // stop when scan returns zero elements
        if (scanFunction.getNumProcessed() == 0) {
            break;
        }
        batches.add(Iterables.transform(scanFunction.getValues(), runRecordMeta -> RunIds.fromString(runRecordMeta.getPid())));
        // key for next scan is the last key + 1 from the previous scan
        startKey = new MDSKey(Bytes.stopKeyForPrefix(scanFunction.getLastKey().getKey()));
    }
    return batches;
}

Example 68 with MDSKey

use of io.cdap.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method getNonCompleteRuns.

private Map<ProgramRunId, RunRecordMeta> getNonCompleteRuns(@Nullable ProgramId programId, String recordType, final long startTime, final long endTime, int limit, Predicate<RunRecordMeta> filter) {
    Predicate<RunRecordMeta> valuePredicate = andPredicate(getTimeRangePredicate(startTime, endTime), filter);
    if (programId == null || !programId.getVersion().equals(ApplicationId.DEFAULT_VERSION)) {
        MDSKey key = getProgramKeyBuilder(recordType, programId).build();
        return getProgramRunIdMap(listKV(key, null, RunRecordMeta.class, limit, valuePredicate));
    }
    Predicate<MDSKey> keyPredicate = new AppVersionPredicate(ApplicationId.DEFAULT_VERSION);
    MDSKey key = getProgramKeyBuilder(recordType, programId).build();
    Map<MDSKey, RunRecordMeta> newRecords = listKV(key, null, RunRecordMeta.class, limit, keyPredicate, valuePredicate);
    int remaining = limit - newRecords.size();
    if (remaining > 0 && !upgradeComplete.get()) {
        // We need to scan twice since the scan key is modified based on whether we include the app version or not.
        key = getVersionLessProgramKeyBuilder(recordType, programId).build();
        Map<MDSKey, RunRecordMeta> oldRecords = listKV(key, null, RunRecordMeta.class, remaining, keyPredicate, valuePredicate);
        newRecords.putAll(oldRecords);
    }
    return getProgramRunIdMap(newRecords);
}

Example 69 with MDSKey

use of io.cdap.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method deleteApplication.

public void deleteApplication(String namespaceId, String appId, String versionId) {
    if (!upgradeComplete.get() && versionId.equals(ApplicationId.DEFAULT_VERSION)) {
        MDSKey mdsKey = new MDSKey.Builder().add(TYPE_APP_META, namespaceId, appId).build();
        ApplicationMeta appMeta = get(mdsKey, ApplicationMeta.class);
        // If app meta exists for the application without a version, delete only that key.
        if (appMeta != null) {
            delete(mdsKey);
        }
    }
    deleteAll(new MDSKey.Builder().add(TYPE_APP_META, namespaceId, appId, versionId).build());
}

Example 70 with MDSKey

use of io.cdap.cdap.data2.dataset2.lib.table.MDSKey in project cdap by caskdata.

the class AppMetadataStore method upgradeVersionKeys.

/**
 * Upgrades the rowkeys for the given record type.
 *
 * @param recordType type of the record
 * @param typeOfT    content type of the record
 * @param <T>        type param
 * @param maxRows    maximum number of rows to be upgraded in this call.
 * @return true if no rows required an upgrade
 */
private <T> boolean upgradeVersionKeys(String recordType, Type typeOfT, int maxRows) {
    LOG.info("Checking upgrade for {}", recordType);
    MDSKey startKey = new MDSKey.Builder().add(recordType).build();
    Map<MDSKey, T> oldMap = listKV(startKey, typeOfT);
    Map<MDSKey, T> newMap = new HashMap<>();
    Set<MDSKey> deleteKeys = new HashSet<>();
    for (Map.Entry<MDSKey, T> oldEntry : oldMap.entrySet()) {
        MDSKey oldKey = oldEntry.getKey();
        MDSKey newKey = appendDefaultVersion(recordType, oldKey);
        // If the key has been modified, only then add it to the map.
        if (!newKey.equals(oldKey)) {
            deleteKeys.add(oldKey);
            // If a row with the new key doesn't exists, only then upgrade the old key otherwise just delete the old key.
            Object valueOfNewKey = get(newKey, typeOfT);
            if (valueOfNewKey == null) {
                newMap.put(newKey, oldEntry.getValue());
            }
            // We want to modify only certain number of rows
            if (deleteKeys.size() >= maxRows) {
                break;
            }
        }
    }
    // No rows needs to be modified
    if (deleteKeys.size() == 0) {
        return true;
    }
    LOG.info("Upgrading {} entries, deleting {} entries of {}", newMap.size(), deleteKeys.size(), recordType);
    // Delete old keys
    for (MDSKey oldKey : deleteKeys) {
        delete(oldKey);
    }
    // Write new rows
    for (Map.Entry<MDSKey, T> newEntry : newMap.entrySet()) {
        write(newEntry.getKey(), newEntry.getValue());
    }
    return false;
}