Examples with ApplicationId org.apache.hadoop.yarn.api.records.ApplicationId used on opensource projects

Example 21 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class TestEntityGroupFSTimelineStore method setup.

@Before
public void setup() throws Exception {
    for (ApplicationId appId : sampleAppIds) {
        Path attemotDirPath = new Path(getTestRootPath(appId.toString()), getAttemptDirName(appId));
        createTestFiles(appId, attemotDirPath);
    }
    store = new EntityGroupFSTimelineStore();
    if (currTestName.getMethodName().contains("Plugin")) {
        rootDir = GenericTestUtils.getTestDir(getClass().getSimpleName());
        if (!rootDir.exists()) {
            rootDir.mkdirs();
        }
        testJar = null;
        testJar = JarFinder.makeClassLoaderTestJar(this.getClass(), rootDir, "test-runjar.jar", 2048, EntityGroupPlugInForTest.class.getName());
        config.set(YarnConfiguration.TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSPATH, testJar.getAbsolutePath());
        // add "-org.apache.hadoop." as system classes
        String systemClasses = "-org.apache.hadoop." + "," + ApplicationClassLoader.SYSTEM_CLASSES_DEFAULT;
        config.set(YarnConfiguration.TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_SYSTEM_CLASSES, systemClasses);
        config.set(YarnConfiguration.TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSES, EntityGroupPlugInForTest.class.getName());
    }
    store.init(config);
    store.setFs(fs);
    store.start();
}

Example 22 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class TestEntityGroupFSTimelineStore method setupClass.

@BeforeClass
public static void setupClass() throws Exception {
    config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_TTL_ENABLE, false);
    config.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES, "YARN_APPLICATION,YARN_APPLICATION_ATTEMPT,YARN_CONTAINER");
    config.setInt(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_APP_CACHE_SIZE, CACHE_TEST_CACHE_SIZE);
    config.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR.toString());
    HdfsConfiguration hdfsConfig = new HdfsConfiguration();
    hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig).numDataNodes(1).build();
    fs = hdfsCluster.getFileSystem();
    fc = FileContext.getFileContext(hdfsCluster.getURI(0), config);
    sampleAppIds = new ArrayList<>(CACHE_TEST_CACHE_SIZE + 1);
    for (int i = 0; i < CACHE_TEST_CACHE_SIZE + 1; i++) {
        ApplicationId appId = ApplicationId.fromString(ConverterUtils.APPLICATION_PREFIX + "_" + SAMPLE_APP_PREFIX_CACHE_TEST + i);
        sampleAppIds.add(appId);
    }
    // Among all sample applicationIds, choose the first one for most of the
    // tests.
    mainTestAppId = sampleAppIds.get(0);
    mainTestAppDirPath = getTestRootPath(mainTestAppId.toString());
    mainEntityLogFileName = EntityGroupFSTimelineStore.ENTITY_LOG_PREFIX + EntityGroupPlugInForTest.getStandardTimelineGroupId(mainTestAppId);
    testDoneDirPath = getTestRootPath("done");
    config.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR, testDoneDirPath.toString());
}

Example 23 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project weave by continuuity.

the class ApplicationMasterProcessLauncher method doLaunch.

@Override
@SuppressWarnings("unchecked")
protected <R> ProcessController<R> doLaunch(YarnLaunchContext launchContext) {
    final ApplicationId appId = getContainerInfo();
    // Set the resource requirement for AM
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(Constants.APP_MASTER_MEMORY_MB);
    YarnUtils.setVirtualCores(capability, 1);
    // Put in extra environments
    Map<String, String> env = ImmutableMap.<String, String>builder().putAll(launchContext.getEnvironment()).put(EnvKeys.YARN_APP_ID, Integer.toString(appId.getId())).put(EnvKeys.YARN_APP_ID_CLUSTER_TIME, Long.toString(appId.getClusterTimestamp())).put(EnvKeys.YARN_APP_ID_STR, appId.toString()).put(EnvKeys.YARN_CONTAINER_MEMORY_MB, Integer.toString(Constants.APP_MASTER_MEMORY_MB)).put(EnvKeys.YARN_CONTAINER_VIRTUAL_CORES, Integer.toString(YarnUtils.getVirtualCores(capability))).build();
    launchContext.setEnvironment(env);
    return (ProcessController<R>) submitter.submit(launchContext, capability);
}

Example 24 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class NMTokenSecretManagerInNM method recover.

public synchronized void recover() throws IOException {
    RecoveredNMTokensState state = stateStore.loadNMTokensState();
    MasterKey key = state.getCurrentMasterKey();
    if (key != null) {
        super.currentMasterKey = new MasterKeyData(key, createSecretKey(key.getBytes().array()));
    }
    key = state.getPreviousMasterKey();
    if (key != null) {
        previousMasterKey = new MasterKeyData(key, createSecretKey(key.getBytes().array()));
    }
    // restore the serial number from the current master key
    if (super.currentMasterKey != null) {
        super.serialNo = super.currentMasterKey.getMasterKey().getKeyId() + 1;
    }
    for (Map.Entry<ApplicationAttemptId, MasterKey> entry : state.getApplicationMasterKeys().entrySet()) {
        key = entry.getValue();
        oldMasterKeys.put(entry.getKey(), new MasterKeyData(key, createSecretKey(key.getBytes().array())));
    }
    // reconstruct app to app attempts map
    appToAppAttemptMap.clear();
    for (ApplicationAttemptId attempt : oldMasterKeys.keySet()) {
        ApplicationId app = attempt.getApplicationId();
        List<ApplicationAttemptId> attempts = appToAppAttemptMap.get(app);
        if (attempts == null) {
            attempts = new ArrayList<ApplicationAttemptId>();
            appToAppAttemptMap.put(app, attempts);
        }
        attempts.add(attempt);
    }
}

Example 25 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class NMLeveldbStateStoreService method loadUserLocalizedResources.

private RecoveredUserResources loadUserLocalizedResources(LeveldbIterator iter, String keyPrefix) throws IOException {
    RecoveredUserResources userResources = new RecoveredUserResources();
    while (iter.hasNext()) {
        Entry<byte[], byte[]> entry = iter.peekNext();
        String key = asString(entry.getKey());
        if (!key.startsWith(keyPrefix)) {
            break;
        }
        if (key.startsWith(LOCALIZATION_FILECACHE_SUFFIX, keyPrefix.length())) {
            userResources.privateTrackerState = loadResourceTrackerState(iter, keyPrefix + LOCALIZATION_FILECACHE_SUFFIX);
        } else if (key.startsWith(LOCALIZATION_APPCACHE_SUFFIX, keyPrefix.length())) {
            int appIdStartPos = keyPrefix.length() + LOCALIZATION_APPCACHE_SUFFIX.length();
            int appIdEndPos = key.indexOf('/', appIdStartPos);
            if (appIdEndPos < 0) {
                throw new IOException("Unable to determine appID in resource key: " + key);
            }
            ApplicationId appId = ApplicationId.fromString(key.substring(appIdStartPos, appIdEndPos));
            userResources.appTrackerStates.put(appId, loadResourceTrackerState(iter, key.substring(0, appIdEndPos + 1)));
        } else {
            throw new IOException("Unexpected user resource key " + key);
        }
    }
    return userResources;
}