Examples with ApplicationId org.apache.hadoop.yarn.api.records.ApplicationId used on opensource projects

Example 96 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class NMLeveldbStateStoreService method loadUserLocalizedResources.

private RecoveredUserResources loadUserLocalizedResources(LeveldbIterator iter, String keyPrefix) throws IOException {
    RecoveredUserResources userResources = new RecoveredUserResources();
    while (iter.hasNext()) {
        Entry<byte[], byte[]> entry = iter.peekNext();
        String key = asString(entry.getKey());
        if (!key.startsWith(keyPrefix)) {
            break;
        }
        if (key.startsWith(LOCALIZATION_FILECACHE_SUFFIX, keyPrefix.length())) {
            userResources.privateTrackerState = loadResourceTrackerState(iter, keyPrefix + LOCALIZATION_FILECACHE_SUFFIX);
        } else if (key.startsWith(LOCALIZATION_APPCACHE_SUFFIX, keyPrefix.length())) {
            int appIdStartPos = keyPrefix.length() + LOCALIZATION_APPCACHE_SUFFIX.length();
            int appIdEndPos = key.indexOf('/', appIdStartPos);
            if (appIdEndPos < 0) {
                throw new IOException("Unable to determine appID in resource key: " + key);
            }
            ApplicationId appId = ApplicationId.fromString(key.substring(appIdStartPos, appIdEndPos));
            userResources.appTrackerStates.put(appId, loadResourceTrackerState(iter, key.substring(0, appIdEndPos + 1)));
        } else {
            throw new IOException("Unexpected user resource key " + key);
        }
    }
    return userResources;
}

Example 97 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class NMLeveldbStateStoreService method loadLogDeleterState.

@Override
public RecoveredLogDeleterState loadLogDeleterState() throws IOException {
    RecoveredLogDeleterState state = new RecoveredLogDeleterState();
    state.logDeleterMap = new HashMap<ApplicationId, LogDeleterProto>();
    LeveldbIterator iter = null;
    try {
        iter = new LeveldbIterator(db);
        iter.seek(bytes(LOG_DELETER_KEY_PREFIX));
        final int logDeleterKeyPrefixLength = LOG_DELETER_KEY_PREFIX.length();
        while (iter.hasNext()) {
            Entry<byte[], byte[]> entry = iter.next();
            String fullKey = asString(entry.getKey());
            if (!fullKey.startsWith(LOG_DELETER_KEY_PREFIX)) {
                break;
            }
            String appIdStr = fullKey.substring(logDeleterKeyPrefixLength);
            ApplicationId appId = null;
            try {
                appId = ApplicationId.fromString(appIdStr);
            } catch (IllegalArgumentException e) {
                LOG.warn("Skipping unknown log deleter key " + fullKey);
                continue;
            }
            LogDeleterProto proto = LogDeleterProto.parseFrom(entry.getValue());
            state.logDeleterMap.put(appId, proto);
        }
    } catch (DBException e) {
        throw new IOException(e);
    } finally {
        if (iter != null) {
            iter.close();
        }
    }
    return state;
}

Example 98 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class AMLauncher method setupTokens.

@Private
@VisibleForTesting
protected void setupTokens(ContainerLaunchContext container, ContainerId containerID) throws IOException {
    Map<String, String> environment = container.getEnvironment();
    environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV, application.getWebProxyBase());
    // Set AppSubmitTime to be consumable by the AM.
    ApplicationId applicationId = application.getAppAttemptId().getApplicationId();
    environment.put(ApplicationConstants.APP_SUBMIT_TIME_ENV, String.valueOf(rmContext.getRMApps().get(applicationId).getSubmitTime()));
    Credentials credentials = new Credentials();
    DataInputByteBuffer dibb = new DataInputByteBuffer();
    ByteBuffer tokens = container.getTokens();
    if (tokens != null) {
        // TODO: Don't do this kind of checks everywhere.
        dibb.reset(tokens);
        credentials.readTokenStorageStream(dibb);
        tokens.rewind();
    }
    // Add AMRMToken
    Token<AMRMTokenIdentifier> amrmToken = createAndSetAMRMToken();
    if (amrmToken != null) {
        credentials.addToken(amrmToken.getService(), amrmToken);
    }
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    container.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength()));
}

Example 99 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class AMLauncher method setFlowContext.

private void setFlowContext(ContainerLaunchContext container) {
    if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
        Map<String, String> environment = container.getEnvironment();
        ApplicationId applicationId = application.getAppAttemptId().getApplicationId();
        RMApp app = rmContext.getRMApps().get(applicationId);
        // initialize the flow in the environment with default values for those
        // that do not specify the flow tags
        // flow name: app name (or app id if app name is missing),
        // flow version: "1", flow run id: start time
        setFlowTags(environment, TimelineUtils.FLOW_NAME_TAG_PREFIX, TimelineUtils.generateDefaultFlowName(app.getName(), applicationId));
        setFlowTags(environment, TimelineUtils.FLOW_VERSION_TAG_PREFIX, TimelineUtils.DEFAULT_FLOW_VERSION);
        setFlowTags(environment, TimelineUtils.FLOW_RUN_ID_TAG_PREFIX, String.valueOf(app.getStartTime()));
        // tags
        for (String tag : app.getApplicationTags()) {
            String[] parts = tag.split(":", 2);
            if (parts.length != 2 || parts[1].isEmpty()) {
                continue;
            }
            switch(parts[0].toUpperCase()) {
                case TimelineUtils.FLOW_NAME_TAG_PREFIX:
                    setFlowTags(environment, TimelineUtils.FLOW_NAME_TAG_PREFIX, parts[1]);
                    break;
                case TimelineUtils.FLOW_VERSION_TAG_PREFIX:
                    setFlowTags(environment, TimelineUtils.FLOW_VERSION_TAG_PREFIX, parts[1]);
                    break;
                case TimelineUtils.FLOW_RUN_ID_TAG_PREFIX:
                    setFlowTags(environment, TimelineUtils.FLOW_RUN_ID_TAG_PREFIX, parts[1]);
                    break;
                default:
                    break;
            }
        }
    }
}

Example 100 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class LeveldbRMStateStore method loadRMApp.

private int loadRMApp(RMState rmState, LeveldbIterator iter, String appIdStr, byte[] appData) throws IOException {
    ApplicationStateData appState = createApplicationState(appIdStr, appData);
    ApplicationId appId = appState.getApplicationSubmissionContext().getApplicationId();
    rmState.appState.put(appId, appState);
    String attemptNodePrefix = getApplicationNodeKey(appId) + SEPARATOR;
    while (iter.hasNext()) {
        Entry<byte[], byte[]> entry = iter.peekNext();
        String key = asString(entry.getKey());
        if (!key.startsWith(attemptNodePrefix)) {
            break;
        }
        String attemptId = key.substring(attemptNodePrefix.length());
        if (attemptId.startsWith(ApplicationAttemptId.appAttemptIdStrPrefix)) {
            ApplicationAttemptStateData attemptState = createAttemptState(attemptId, entry.getValue());
            appState.attempts.put(attemptState.getAttemptId(), attemptState);
        } else {
            LOG.warn("Ignoring unknown application key: " + key);
        }
        iter.next();
    }
    int numAttempts = appState.attempts.size();
    if (LOG.isDebugEnabled()) {
        LOG.debug("Loaded application " + appId + " with " + numAttempts + " attempts");
    }
    return numAttempts;
}