Examples with ApplicationId org.apache.hadoop.yarn.api.records.ApplicationId used on opensource projects

Example 26 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class NMLeveldbStateStoreService method loadLogDeleterState.

@Override
public RecoveredLogDeleterState loadLogDeleterState() throws IOException {
    RecoveredLogDeleterState state = new RecoveredLogDeleterState();
    state.logDeleterMap = new HashMap<ApplicationId, LogDeleterProto>();
    LeveldbIterator iter = null;
    try {
        iter = new LeveldbIterator(db);
        iter.seek(bytes(LOG_DELETER_KEY_PREFIX));
        final int logDeleterKeyPrefixLength = LOG_DELETER_KEY_PREFIX.length();
        while (iter.hasNext()) {
            Entry<byte[], byte[]> entry = iter.next();
            String fullKey = asString(entry.getKey());
            if (!fullKey.startsWith(LOG_DELETER_KEY_PREFIX)) {
                break;
            }
            String appIdStr = fullKey.substring(logDeleterKeyPrefixLength);
            ApplicationId appId = null;
            try {
                appId = ApplicationId.fromString(appIdStr);
            } catch (IllegalArgumentException e) {
                LOG.warn("Skipping unknown log deleter key " + fullKey);
                continue;
            }
            LogDeleterProto proto = LogDeleterProto.parseFrom(entry.getValue());
            state.logDeleterMap.put(appId, proto);
        }
    } catch (DBException e) {
        throw new IOException(e);
    } finally {
        if (iter != null) {
            iter.close();
        }
    }
    return state;
}

Example 27 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class AMLauncher method setupTokens.

@Private
@VisibleForTesting
protected void setupTokens(ContainerLaunchContext container, ContainerId containerID) throws IOException {
    Map<String, String> environment = container.getEnvironment();
    environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV, application.getWebProxyBase());
    // Set AppSubmitTime to be consumable by the AM.
    ApplicationId applicationId = application.getAppAttemptId().getApplicationId();
    environment.put(ApplicationConstants.APP_SUBMIT_TIME_ENV, String.valueOf(rmContext.getRMApps().get(applicationId).getSubmitTime()));
    Credentials credentials = new Credentials();
    DataInputByteBuffer dibb = new DataInputByteBuffer();
    ByteBuffer tokens = container.getTokens();
    if (tokens != null) {
        // TODO: Don't do this kind of checks everywhere.
        dibb.reset(tokens);
        credentials.readTokenStorageStream(dibb);
        tokens.rewind();
    }
    // Add AMRMToken
    Token<AMRMTokenIdentifier> amrmToken = createAndSetAMRMToken();
    if (amrmToken != null) {
        credentials.addToken(amrmToken.getService(), amrmToken);
    }
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    container.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength()));
}

Example 28 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class AMLauncher method setFlowContext.

private void setFlowContext(ContainerLaunchContext container) {
    if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
        Map<String, String> environment = container.getEnvironment();
        ApplicationId applicationId = application.getAppAttemptId().getApplicationId();
        RMApp app = rmContext.getRMApps().get(applicationId);
        // initialize the flow in the environment with default values for those
        // that do not specify the flow tags
        // flow name: app name (or app id if app name is missing),
        // flow version: "1", flow run id: start time
        setFlowTags(environment, TimelineUtils.FLOW_NAME_TAG_PREFIX, TimelineUtils.generateDefaultFlowName(app.getName(), applicationId));
        setFlowTags(environment, TimelineUtils.FLOW_VERSION_TAG_PREFIX, TimelineUtils.DEFAULT_FLOW_VERSION);
        setFlowTags(environment, TimelineUtils.FLOW_RUN_ID_TAG_PREFIX, String.valueOf(app.getStartTime()));
        // tags
        for (String tag : app.getApplicationTags()) {
            String[] parts = tag.split(":", 2);
            if (parts.length != 2 || parts[1].isEmpty()) {
                continue;
            }
            switch(parts[0].toUpperCase()) {
                case TimelineUtils.FLOW_NAME_TAG_PREFIX:
                    setFlowTags(environment, TimelineUtils.FLOW_NAME_TAG_PREFIX, parts[1]);
                    break;
                case TimelineUtils.FLOW_VERSION_TAG_PREFIX:
                    setFlowTags(environment, TimelineUtils.FLOW_VERSION_TAG_PREFIX, parts[1]);
                    break;
                case TimelineUtils.FLOW_RUN_ID_TAG_PREFIX:
                    setFlowTags(environment, TimelineUtils.FLOW_RUN_ID_TAG_PREFIX, parts[1]);
                    break;
                default:
                    break;
            }
        }
    }
}

Example 29 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class LeveldbRMStateStore method loadRMApp.

private int loadRMApp(RMState rmState, LeveldbIterator iter, String appIdStr, byte[] appData) throws IOException {
    ApplicationStateData appState = createApplicationState(appIdStr, appData);
    ApplicationId appId = appState.getApplicationSubmissionContext().getApplicationId();
    rmState.appState.put(appId, appState);
    String attemptNodePrefix = getApplicationNodeKey(appId) + SEPARATOR;
    while (iter.hasNext()) {
        Entry<byte[], byte[]> entry = iter.peekNext();
        String key = asString(entry.getKey());
        if (!key.startsWith(attemptNodePrefix)) {
            break;
        }
        String attemptId = key.substring(attemptNodePrefix.length());
        if (attemptId.startsWith(ApplicationAttemptId.appAttemptIdStrPrefix)) {
            ApplicationAttemptStateData attemptState = createAttemptState(attemptId, entry.getValue());
            appState.attempts.put(attemptState.getAttemptId(), attemptState);
        } else {
            LOG.warn("Ignoring unknown application key: " + key);
        }
        iter.next();
    }
    int numAttempts = appState.attempts.size();
    if (LOG.isDebugEnabled()) {
        LOG.debug("Loaded application " + appId + " with " + numAttempts + " attempts");
    }
    return numAttempts;
}

Example 30 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class MemoryRMStateStore method removeApplicationStateInternal.

@Override
public synchronized void removeApplicationStateInternal(ApplicationStateData appState) throws Exception {
    ApplicationId appId = appState.getApplicationSubmissionContext().getApplicationId();
    ApplicationStateData removed = state.appState.remove(appId);
    if (removed == null) {
        throw new YarnRuntimeException("Removing non-existing application state");
    }
}