Examples with ContainerLaunchParameters com.cloudera.kitten.ContainerLaunchParameters used on opensource projects

Example 1 with ContainerLaunchParameters

use of com.cloudera.kitten.ContainerLaunchParameters in project kitten by cloudera.

the class YarnClientServiceImpl method startUp.

@Override
protected void startUp() throws IOException {
    ByteBuffer serializedTokens = null;
    if (UserGroupInformation.isSecurityEnabled()) {
        Configuration conf = this.yarnClientFactory.getConfig();
        FileSystem fs = FileSystem.get(conf);
        Credentials credentials = new Credentials();
        String tokenRenewer = this.yarnClientFactory.getConfig().get(YarnConfiguration.RM_PRINCIPAL);
        if (tokenRenewer == null || tokenRenewer.length() == 0) {
            throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
        }
        // For now, only getting tokens for the default file-system.
        final Token<?>[] tokens = fs.addDelegationTokens(tokenRenewer, credentials);
        if (tokens != null) {
            for (Token<?> token : tokens) {
                LOG.info("Got delegation token for " + fs.getUri() + "; " + token);
            }
        }
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        serializedTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    }
    this.yarnClient = yarnClientFactory.connect();
    YarnClientApplication clientApp = getNewApplication();
    GetNewApplicationResponse newApp = clientApp.getNewApplicationResponse();
    ContainerLaunchContextFactory clcFactory = new ContainerLaunchContextFactory(newApp.getMaximumResourceCapability(), serializedTokens);
    ApplicationSubmissionContext appContext = clientApp.getApplicationSubmissionContext();
    this.applicationId = appContext.getApplicationId();
    appContext.setApplicationName(parameters.getApplicationName());
    // Setup the container for the application master.
    ContainerLaunchParameters appMasterParams = parameters.getApplicationMasterParameters(applicationId);
    ContainerLaunchContext clc = clcFactory.create(appMasterParams);
    LOG.debug("Master context: " + clc);
    appContext.setResource(clcFactory.createResource(appMasterParams));
    appContext.setQueue(parameters.getQueue());
    appContext.setPriority(clcFactory.createPriority(appMasterParams.getPriority()));
    appContext.setAMContainerSpec(clc);
    submitApplication(appContext);
    // Make sure we stop the application in the case that it isn't done already.
    Runtime.getRuntime().addShutdownHook(new Thread() {

        @Override
        public void run() {
            if (YarnClientServiceImpl.this.isRunning()) {
                YarnClientServiceImpl.this.stop();
            }
        }
    });
    stopwatch.start();
}

Example 2 with ContainerLaunchParameters

use of com.cloudera.kitten.ContainerLaunchParameters in project kitten by cloudera.

the class LuaApplicationMasterParameters method getContainerLaunchParameters.

@Override
public List<ContainerLaunchParameters> getContainerLaunchParameters() {
    if (!env.isNil(LuaFields.CONTAINERS)) {
        List<ContainerLaunchParameters> clp = Lists.newArrayList();
        Iterator<LuaPair> iter = env.getTable(LuaFields.CONTAINERS).arrayIterator();
        while (iter.hasNext()) {
            clp.add(new LuaContainerLaunchParameters(iter.next().value, conf, localToUris));
        }
        return clp;
    } else if (!env.isNil(LuaFields.CONTAINER)) {
        return ImmutableList.<ContainerLaunchParameters>of(new LuaContainerLaunchParameters(env.getTable(LuaFields.CONTAINER), conf, localToUris));
    }
    return ImmutableList.of();
}

Example 3 with ContainerLaunchParameters

use of com.cloudera.kitten.ContainerLaunchParameters in project kitten by cloudera.

the class BasicLuaConfigTest method testBasicClient.

@Test
public void testBasicClient() throws Exception {
    File tmpFile = File.createTempFile("kitten", ".lua");
    Files.copy(newInputStreamSupplier(getResource("lua/test1.lua")), tmpFile);
    tmpFile.deleteOnExit();
    YarnClientParameters params = new LuaYarnClientParameters(tmpFile.getAbsolutePath(), "distshell", conf);
    assertEquals("Distributed Shell", params.getApplicationName());
    assertEquals(86400L, params.getClientTimeoutMillis());
    assertEquals("default", params.getQueue());
    ContainerLaunchParameters clp = params.getApplicationMasterParameters(null);
    assertEquals(1, clp.getPriority());
    // clusterMax = 90 < 100
    assertEquals(clusterMax, clp.getContainerResource(clusterMax));
    Map<String, String> expEnv = ImmutableMap.of("zs", "10", "a", "b", "fiz", "faz", "foo", "foo", "biz", "baz");
    Map<String, String> actEnv = clp.getEnvironment();
    for (Map.Entry<String, String> e : expEnv.entrySet()) {
        assertEquals(e.getValue(), actEnv.get(e.getKey()));
    }
}

Example 4 with ContainerLaunchParameters

use of com.cloudera.kitten.ContainerLaunchParameters in project kitten by cloudera.

the class ApplicationMasterServiceImpl method startUp.

@Override
protected void startUp() throws IOException {
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    // Now remove the AM->RM token so that containers cannot access it.
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    LOG.info("Executing with tokens:");
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        LOG.info(token);
        if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            iter.remove();
        }
    }
    ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    // Create appSubmitterUgi and add original tokens to it
    String userName = System.getenv(ApplicationConstants.Environment.USER.name());
    appSubmitterUgi = UserGroupInformation.createRemoteUser(userName);
    appSubmitterUgi.addCredentials(credentials);
    this.resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, this);
    this.resourceManager.init(conf);
    this.resourceManager.start();
    RegisterApplicationMasterResponse registration;
    try {
        registration = resourceManager.registerApplicationMaster(parameters.getHostname(), parameters.getClientPort(), parameters.getTrackingUrl());
    } catch (Exception e) {
        LOG.error("Exception thrown registering application master", e);
        stop();
        return;
    }
    ContainerLaunchContextFactory factory = new ContainerLaunchContextFactory(registration.getMaximumResourceCapability(), tokens);
    for (ContainerLaunchParameters clp : parameters.getContainerLaunchParameters()) {
        ContainerTracker tracker = new ContainerTracker(clp);
        tracker.init(factory);
        trackers.add(tracker);
    }
    this.hasRunningContainers = true;
}