Examples with LocalResource org.apache.hadoop.yarn.api.records.LocalResource used on opensource projects

Example 76 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project flink by apache.

the class AbstractYarnClusterDescriptor method startAppMaster.

public ApplicationReport startAppMaster(JobGraph jobGraph, YarnClient yarnClient, YarnClientApplication yarnApplication) throws Exception {
    try {
        org.apache.flink.core.fs.FileSystem.setDefaultScheme(flinkConfiguration);
    } catch (IOException e) {
        throw new IOException("Error while setting the default " + "filesystem scheme from configuration.", e);
    }
    // initialize file system
    // Copy the application master jar to the filesystem
    // Create a local resource to point to the destination jar path
    final FileSystem fs = FileSystem.get(conf);
    // hard coded check for the GoogleHDFS client because its not overriding the getScheme() method.
    if (!fs.getClass().getSimpleName().equals("GoogleHadoopFileSystem") && fs.getScheme().startsWith("file")) {
        LOG.warn("The file system scheme is '" + fs.getScheme() + "'. This indicates that the " + "specified Hadoop configuration path is wrong and the system is using the default Hadoop configuration values." + "The Flink YARN client needs to store its files in a distributed file system");
    }
    ApplicationSubmissionContext appContext = yarnApplication.getApplicationSubmissionContext();
    Set<File> effectiveShipFiles = new HashSet<>(shipFiles.size());
    for (File file : shipFiles) {
        effectiveShipFiles.add(file.getAbsoluteFile());
    }
    //check if there is a logback or log4j file
    File logbackFile = new File(configurationDirectory + File.separator + CONFIG_FILE_LOGBACK_NAME);
    final boolean hasLogback = logbackFile.exists();
    if (hasLogback) {
        effectiveShipFiles.add(logbackFile);
    }
    File log4jFile = new File(configurationDirectory + File.separator + CONFIG_FILE_LOG4J_NAME);
    final boolean hasLog4j = log4jFile.exists();
    if (hasLog4j) {
        effectiveShipFiles.add(log4jFile);
        if (hasLogback) {
            // this means there is already a logback configuration file --> fail
            LOG.warn("The configuration directory ('" + configurationDirectory + "') contains both LOG4J and " + "Logback configuration files. Please delete or rename one of them.");
        }
    }
    addLibFolderToShipFiles(effectiveShipFiles);
    // add the user jar to the classpath of the to-be-created cluster
    if (userJarFiles != null) {
        effectiveShipFiles.addAll(userJarFiles);
    }
    // Set-up ApplicationSubmissionContext for the application
    final ApplicationId appId = appContext.getApplicationId();
    // ------------------ Add Zookeeper namespace to local flinkConfiguraton ------
    String zkNamespace = getZookeeperNamespace();
    // no user specified cli argument for namespace?
    if (zkNamespace == null || zkNamespace.isEmpty()) {
        // namespace defined in config? else use applicationId as default.
        zkNamespace = flinkConfiguration.getString(HighAvailabilityOptions.HA_CLUSTER_ID, String.valueOf(appId));
        setZookeeperNamespace(zkNamespace);
    }
    flinkConfiguration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zkNamespace);
    if (HighAvailabilityMode.isHighAvailabilityModeActivated(flinkConfiguration)) {
        // activate re-execution of failed applications
        appContext.setMaxAppAttempts(flinkConfiguration.getInteger(ConfigConstants.YARN_APPLICATION_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS));
        activateHighAvailabilitySupport(appContext);
    } else {
        // set number of application retries to 1 in the default case
        appContext.setMaxAppAttempts(flinkConfiguration.getInteger(ConfigConstants.YARN_APPLICATION_ATTEMPTS, 1));
    }
    // local resource map for Yarn
    final Map<String, LocalResource> localResources = new HashMap<>(2 + effectiveShipFiles.size());
    // list of remote paths (after upload)
    final List<Path> paths = new ArrayList<>(2 + effectiveShipFiles.size());
    // classpath assembler
    final StringBuilder classPathBuilder = new StringBuilder();
    // ship list that enables reuse of resources for task manager containers
    StringBuilder envShipFileList = new StringBuilder();
    // upload and register ship files
    for (File shipFile : effectiveShipFiles) {
        LocalResource shipResources = Records.newRecord(LocalResource.class);
        Path shipLocalPath = new Path("file://" + shipFile.getAbsolutePath());
        Path remotePath = Utils.setupLocalResource(fs, appId.toString(), shipLocalPath, shipResources, fs.getHomeDirectory());
        paths.add(remotePath);
        localResources.put(shipFile.getName(), shipResources);
        if (shipFile.isDirectory()) {
            // add directories to the classpath
            java.nio.file.Path shipPath = shipFile.toPath();
            final java.nio.file.Path parentPath = shipPath.getParent();
            Files.walkFileTree(shipPath, new SimpleFileVisitor<java.nio.file.Path>() {

                @Override
                public FileVisitResult preVisitDirectory(java.nio.file.Path dir, BasicFileAttributes attrs) throws IOException {
                    super.preVisitDirectory(dir, attrs);
                    java.nio.file.Path relativePath = parentPath.relativize(dir);
                    classPathBuilder.append(relativePath).append(File.separator).append("*").append(File.pathSeparator);
                    return FileVisitResult.CONTINUE;
                }
            });
        } else {
            // add files to the classpath
            classPathBuilder.append(shipFile.getName()).append(File.pathSeparator);
        }
        envShipFileList.append(remotePath).append(",");
    }
    // Setup jar for ApplicationMaster
    LocalResource appMasterJar = Records.newRecord(LocalResource.class);
    LocalResource flinkConf = Records.newRecord(LocalResource.class);
    Path remotePathJar = Utils.setupLocalResource(fs, appId.toString(), flinkJarPath, appMasterJar, fs.getHomeDirectory());
    Path remotePathConf = Utils.setupLocalResource(fs, appId.toString(), flinkConfigurationPath, flinkConf, fs.getHomeDirectory());
    localResources.put("flink.jar", appMasterJar);
    localResources.put("flink-conf.yaml", flinkConf);
    paths.add(remotePathJar);
    classPathBuilder.append("flink.jar").append(File.pathSeparator);
    paths.add(remotePathConf);
    classPathBuilder.append("flink-conf.yaml").append(File.pathSeparator);
    // TODO: server use user main method to generate job graph
    if (jobGraph != null) {
        try {
            File fp = File.createTempFile(appId.toString(), null);
            fp.deleteOnExit();
            try (FileOutputStream output = new FileOutputStream(fp);
                ObjectOutputStream obOutput = new ObjectOutputStream(output)) {
                obOutput.writeObject(jobGraph);
            }
            LocalResource jobgraph = Records.newRecord(LocalResource.class);
            Path remoteJobGraph = Utils.setupLocalResource(fs, appId.toString(), new Path(fp.toURI()), jobgraph, fs.getHomeDirectory());
            localResources.put("job.graph", jobgraph);
            paths.add(remoteJobGraph);
            classPathBuilder.append("job.graph").append(File.pathSeparator);
        } catch (Exception e) {
            LOG.warn("Add job graph to local resource fail");
            throw e;
        }
    }
    sessionFilesDir = new Path(fs.getHomeDirectory(), ".flink/" + appId.toString() + "/");
    FsPermission permission = new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE);
    // set permission for path.
    fs.setPermission(sessionFilesDir, permission);
    //To support Yarn Secure Integration Test Scenario
    //In Integration test setup, the Yarn containers created by YarnMiniCluster does not have the Yarn site XML
    //and KRB5 configuration files. We are adding these files as container local resources for the container
    //applications (JM/TMs) to have proper secure cluster setup
    Path remoteKrb5Path = null;
    Path remoteYarnSiteXmlPath = null;
    boolean hasKrb5 = false;
    if (System.getenv("IN_TESTS") != null) {
        String krb5Config = System.getProperty("java.security.krb5.conf");
        if (krb5Config != null && krb5Config.length() != 0) {
            File krb5 = new File(krb5Config);
            LOG.info("Adding KRB5 configuration {} to the AM container local resource bucket", krb5.getAbsolutePath());
            LocalResource krb5ConfResource = Records.newRecord(LocalResource.class);
            Path krb5ConfPath = new Path(krb5.getAbsolutePath());
            remoteKrb5Path = Utils.setupLocalResource(fs, appId.toString(), krb5ConfPath, krb5ConfResource, fs.getHomeDirectory());
            localResources.put(Utils.KRB5_FILE_NAME, krb5ConfResource);
            File f = new File(System.getenv("YARN_CONF_DIR"), Utils.YARN_SITE_FILE_NAME);
            LOG.info("Adding Yarn configuration {} to the AM container local resource bucket", f.getAbsolutePath());
            LocalResource yarnConfResource = Records.newRecord(LocalResource.class);
            Path yarnSitePath = new Path(f.getAbsolutePath());
            remoteYarnSiteXmlPath = Utils.setupLocalResource(fs, appId.toString(), yarnSitePath, yarnConfResource, fs.getHomeDirectory());
            localResources.put(Utils.YARN_SITE_FILE_NAME, yarnConfResource);
            hasKrb5 = true;
        }
    }
    // setup security tokens
    LocalResource keytabResource = null;
    Path remotePathKeytab = null;
    String keytab = flinkConfiguration.getString(SecurityOptions.KERBEROS_LOGIN_KEYTAB);
    if (keytab != null) {
        LOG.info("Adding keytab {} to the AM container local resource bucket", keytab);
        keytabResource = Records.newRecord(LocalResource.class);
        Path keytabPath = new Path(keytab);
        remotePathKeytab = Utils.setupLocalResource(fs, appId.toString(), keytabPath, keytabResource, fs.getHomeDirectory());
        localResources.put(Utils.KEYTAB_FILE_NAME, keytabResource);
    }
    final ContainerLaunchContext amContainer = setupApplicationMasterContainer(hasLogback, hasLog4j, hasKrb5);
    if (UserGroupInformation.isSecurityEnabled() && keytab == null) {
        //set tokens only when keytab is not provided
        LOG.info("Adding delegation token to the AM container..");
        Utils.setTokensFor(amContainer, paths, conf);
    }
    amContainer.setLocalResources(localResources);
    fs.close();
    // Setup CLASSPATH and environment variables for ApplicationMaster
    final Map<String, String> appMasterEnv = new HashMap<>();
    // set user specified app master environment variables
    appMasterEnv.putAll(Utils.getEnvironmentVariables(ConfigConstants.YARN_APPLICATION_MASTER_ENV_PREFIX, flinkConfiguration));
    // set Flink app class path
    appMasterEnv.put(YarnConfigKeys.ENV_FLINK_CLASSPATH, classPathBuilder.toString());
    // set Flink on YARN internal configuration values
    appMasterEnv.put(YarnConfigKeys.ENV_TM_COUNT, String.valueOf(taskManagerCount));
    appMasterEnv.put(YarnConfigKeys.ENV_TM_MEMORY, String.valueOf(taskManagerMemoryMb));
    appMasterEnv.put(YarnConfigKeys.FLINK_JAR_PATH, remotePathJar.toString());
    appMasterEnv.put(YarnConfigKeys.ENV_APP_ID, appId.toString());
    appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, fs.getHomeDirectory().toString());
    appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, envShipFileList.toString());
    appMasterEnv.put(YarnConfigKeys.ENV_SLOTS, String.valueOf(slots));
    appMasterEnv.put(YarnConfigKeys.ENV_DETACHED, String.valueOf(detached));
    appMasterEnv.put(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE, getZookeeperNamespace());
    // https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md#identity-on-an-insecure-cluster-hadoop_user_name
    appMasterEnv.put(YarnConfigKeys.ENV_HADOOP_USER_NAME, UserGroupInformation.getCurrentUser().getUserName());
    if (keytabResource != null) {
        appMasterEnv.put(YarnConfigKeys.KEYTAB_PATH, remotePathKeytab.toString());
        String principal = flinkConfiguration.getString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL);
        appMasterEnv.put(YarnConfigKeys.KEYTAB_PRINCIPAL, principal);
    }
    //To support Yarn Secure Integration Test Scenario
    if (remoteYarnSiteXmlPath != null && remoteKrb5Path != null) {
        appMasterEnv.put(YarnConfigKeys.ENV_YARN_SITE_XML_PATH, remoteYarnSiteXmlPath.toString());
        appMasterEnv.put(YarnConfigKeys.ENV_KRB5_PATH, remoteKrb5Path.toString());
    }
    if (dynamicPropertiesEncoded != null) {
        appMasterEnv.put(YarnConfigKeys.ENV_DYNAMIC_PROPERTIES, dynamicPropertiesEncoded);
    }
    // set classpath from YARN configuration
    Utils.setupYarnClassPath(conf, appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);
    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(jobManagerMemoryMb);
    capability.setVirtualCores(1);
    String name;
    if (customName == null) {
        name = "Flink session with " + taskManagerCount + " TaskManagers";
        if (detached) {
            name += " (detached)";
        }
    } else {
        name = customName;
    }
    appContext.setApplicationName(name);
    appContext.setApplicationType("Apache Flink");
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    if (yarnQueue != null) {
        appContext.setQueue(yarnQueue);
    }
    setApplicationTags(appContext);
    // add a hook to clean up in case deployment fails
    Thread deploymentFailureHook = new DeploymentFailureHook(yarnClient, yarnApplication);
    Runtime.getRuntime().addShutdownHook(deploymentFailureHook);
    LOG.info("Submitting application master " + appId);
    yarnClient.submitApplication(appContext);
    LOG.info("Waiting for the cluster to be allocated");
    final long startTime = System.currentTimeMillis();
    ApplicationReport report;
    YarnApplicationState lastAppState = YarnApplicationState.NEW;
    loop: while (true) {
        try {
            report = yarnClient.getApplicationReport(appId);
        } catch (IOException e) {
            throw new YarnDeploymentException("Failed to deploy the cluster.", e);
        }
        YarnApplicationState appState = report.getYarnApplicationState();
        LOG.debug("Application State: {}", appState);
        switch(appState) {
            case FAILED:
            //TODO: the finished state may be valid in flip-6
            case FINISHED:
            case KILLED:
                throw new YarnDeploymentException("The YARN application unexpectedly switched to state " + appState + " during deployment. \n" + "Diagnostics from YARN: " + report.getDiagnostics() + "\n" + "If log aggregation is enabled on your cluster, use this command to further investigate the issue:\n" + "yarn logs -applicationId " + appId);
            //break ..
            case RUNNING:
                LOG.info("YARN application has been deployed successfully.");
                break loop;
            default:
                if (appState != lastAppState) {
                    LOG.info("Deploying cluster, current state " + appState);
                }
                if (System.currentTimeMillis() - startTime > 60000) {
                    LOG.info("Deployment took more than 60 seconds. Please check if the requested resources are available in the YARN cluster");
                }
        }
        lastAppState = appState;
        Thread.sleep(250);
    }
    // print the application id for user to cancel themselves.
    if (isDetachedMode()) {
        LOG.info("The Flink YARN client has been started in detached mode. In order to stop " + "Flink on YARN, use the following command or a YARN web interface to stop " + "it:\nyarn application -kill " + appId + "\nPlease also note that the " + "temporary files of the YARN session in the home directoy will not be removed.");
    }
    // since deployment was successful, remove the hook
    try {
        Runtime.getRuntime().removeShutdownHook(deploymentFailureHook);
    } catch (IllegalStateException e) {
    // we're already in the shut down hook.
    }
    return report;
}

Example 77 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project flink by apache.

the class Utils method createTaskExecutorContext.

/**
	 * Creates the launch context, which describes how to bring up a TaskExecutor / TaskManager process in
	 * an allocated YARN container.
	 *
	 * <p>This code is extremely YARN specific and registers all the resources that the TaskExecutor
	 * needs (such as JAR file, config file, ...) and all environment variables in a YARN
	 * container launch context. The launch context then ensures that those resources will be
	 * copied into the containers transient working directory.
	 *
	 * @param flinkConfig
	 *		 The Flink configuration object.
	 * @param yarnConfig
	 *		 The YARN configuration object.
	 * @param env
	 *		 The environment variables.
	 * @param tmParams
	 *		 The TaskExecutor container memory parameters.
	 * @param taskManagerConfig
	 *		 The configuration for the TaskExecutors.
	 * @param workingDirectory
	 *		 The current application master container's working directory.
	 * @param taskManagerMainClass
	 *		 The class with the main method.
	 * @param log
	 *		 The logger.
	 *
	 * @return The launch context for the TaskManager processes.
	 *
	 * @throws Exception Thrown if teh launch context could not be created, for example if
	 *				   the resources could not be copied.
	 */
static ContainerLaunchContext createTaskExecutorContext(org.apache.flink.configuration.Configuration flinkConfig, YarnConfiguration yarnConfig, Map<String, String> env, ContaineredTaskManagerParameters tmParams, org.apache.flink.configuration.Configuration taskManagerConfig, String workingDirectory, Class<?> taskManagerMainClass, Logger log) throws Exception {
    // get and validate all relevant variables
    String remoteFlinkJarPath = env.get(YarnConfigKeys.FLINK_JAR_PATH);
    require(remoteFlinkJarPath != null, "Environment variable %s not set", YarnConfigKeys.FLINK_JAR_PATH);
    String appId = env.get(YarnConfigKeys.ENV_APP_ID);
    require(appId != null, "Environment variable %s not set", YarnConfigKeys.ENV_APP_ID);
    String clientHomeDir = env.get(YarnConfigKeys.ENV_CLIENT_HOME_DIR);
    require(clientHomeDir != null, "Environment variable %s not set", YarnConfigKeys.ENV_CLIENT_HOME_DIR);
    String shipListString = env.get(YarnConfigKeys.ENV_CLIENT_SHIP_FILES);
    require(shipListString != null, "Environment variable %s not set", YarnConfigKeys.ENV_CLIENT_SHIP_FILES);
    String yarnClientUsername = env.get(YarnConfigKeys.ENV_HADOOP_USER_NAME);
    require(yarnClientUsername != null, "Environment variable %s not set", YarnConfigKeys.ENV_HADOOP_USER_NAME);
    final String remoteKeytabPath = env.get(YarnConfigKeys.KEYTAB_PATH);
    log.info("TM:remote keytab path obtained {}", remoteKeytabPath);
    final String remoteKeytabPrincipal = env.get(YarnConfigKeys.KEYTAB_PRINCIPAL);
    log.info("TM:remote keytab principal obtained {}", remoteKeytabPrincipal);
    final String remoteYarnConfPath = env.get(YarnConfigKeys.ENV_YARN_SITE_XML_PATH);
    log.info("TM:remote yarn conf path obtained {}", remoteYarnConfPath);
    final String remoteKrb5Path = env.get(YarnConfigKeys.ENV_KRB5_PATH);
    log.info("TM:remote krb5 path obtained {}", remoteKrb5Path);
    String classPathString = env.get(ENV_FLINK_CLASSPATH);
    require(classPathString != null, "Environment variable %s not set", YarnConfigKeys.ENV_FLINK_CLASSPATH);
    //register keytab
    LocalResource keytabResource = null;
    if (remoteKeytabPath != null) {
        log.info("Adding keytab {} to the AM container local resource bucket", remoteKeytabPath);
        keytabResource = Records.newRecord(LocalResource.class);
        Path keytabPath = new Path(remoteKeytabPath);
        FileSystem fs = keytabPath.getFileSystem(yarnConfig);
        registerLocalResource(fs, keytabPath, keytabResource);
    }
    //To support Yarn Secure Integration Test Scenario
    LocalResource yarnConfResource = null;
    LocalResource krb5ConfResource = null;
    boolean hasKrb5 = false;
    if (remoteYarnConfPath != null && remoteKrb5Path != null) {
        log.info("TM:Adding remoteYarnConfPath {} to the container local resource bucket", remoteYarnConfPath);
        yarnConfResource = Records.newRecord(LocalResource.class);
        Path yarnConfPath = new Path(remoteYarnConfPath);
        FileSystem fs = yarnConfPath.getFileSystem(yarnConfig);
        registerLocalResource(fs, yarnConfPath, yarnConfResource);
        log.info("TM:Adding remoteKrb5Path {} to the container local resource bucket", remoteKrb5Path);
        krb5ConfResource = Records.newRecord(LocalResource.class);
        Path krb5ConfPath = new Path(remoteKrb5Path);
        fs = krb5ConfPath.getFileSystem(yarnConfig);
        registerLocalResource(fs, krb5ConfPath, krb5ConfResource);
        hasKrb5 = true;
    }
    // register Flink Jar with remote HDFS
    LocalResource flinkJar = Records.newRecord(LocalResource.class);
    {
        Path remoteJarPath = new Path(remoteFlinkJarPath);
        FileSystem fs = remoteJarPath.getFileSystem(yarnConfig);
        registerLocalResource(fs, remoteJarPath, flinkJar);
    }
    // register conf with local fs
    LocalResource flinkConf = Records.newRecord(LocalResource.class);
    {
        // write the TaskManager configuration to a local file
        final File taskManagerConfigFile = new File(workingDirectory, UUID.randomUUID() + "-taskmanager-conf.yaml");
        log.debug("Writing TaskManager configuration to {}", taskManagerConfigFile.getAbsolutePath());
        BootstrapTools.writeConfiguration(taskManagerConfig, taskManagerConfigFile);
        Path homeDirPath = new Path(clientHomeDir);
        FileSystem fs = homeDirPath.getFileSystem(yarnConfig);
        setupLocalResource(fs, appId, new Path(taskManagerConfigFile.toURI()), flinkConf, new Path(clientHomeDir));
        log.info("Prepared local resource for modified yaml: {}", flinkConf);
    }
    Map<String, LocalResource> taskManagerLocalResources = new HashMap<>();
    taskManagerLocalResources.put("flink.jar", flinkJar);
    taskManagerLocalResources.put("flink-conf.yaml", flinkConf);
    //To support Yarn Secure Integration Test Scenario
    if (yarnConfResource != null && krb5ConfResource != null) {
        taskManagerLocalResources.put(YARN_SITE_FILE_NAME, yarnConfResource);
        taskManagerLocalResources.put(KRB5_FILE_NAME, krb5ConfResource);
    }
    if (keytabResource != null) {
        taskManagerLocalResources.put(KEYTAB_FILE_NAME, keytabResource);
    }
    // prepare additional files to be shipped
    for (String pathStr : shipListString.split(",")) {
        if (!pathStr.isEmpty()) {
            LocalResource resource = Records.newRecord(LocalResource.class);
            Path path = new Path(pathStr);
            registerLocalResource(path.getFileSystem(yarnConfig), path, resource);
            taskManagerLocalResources.put(path.getName(), resource);
        }
    }
    // now that all resources are prepared, we can create the launch context
    log.info("Creating container launch context for TaskManagers");
    boolean hasLogback = new File(workingDirectory, "logback.xml").exists();
    boolean hasLog4j = new File(workingDirectory, "log4j.properties").exists();
    String launchCommand = BootstrapTools.getTaskManagerShellCommand(flinkConfig, tmParams, ".", ApplicationConstants.LOG_DIR_EXPANSION_VAR, hasLogback, hasLog4j, hasKrb5, taskManagerMainClass);
    log.info("Starting TaskManagers with command: " + launchCommand);
    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
    ctx.setCommands(Collections.singletonList(launchCommand));
    ctx.setLocalResources(taskManagerLocalResources);
    Map<String, String> containerEnv = new HashMap<>();
    containerEnv.putAll(tmParams.taskManagerEnv());
    // add YARN classpath, etc to the container environment
    containerEnv.put(ENV_FLINK_CLASSPATH, classPathString);
    setupYarnClassPath(yarnConfig, containerEnv);
    containerEnv.put(YarnConfigKeys.ENV_HADOOP_USER_NAME, UserGroupInformation.getCurrentUser().getUserName());
    if (remoteKeytabPath != null && remoteKeytabPrincipal != null) {
        containerEnv.put(YarnConfigKeys.KEYTAB_PATH, remoteKeytabPath);
        containerEnv.put(YarnConfigKeys.KEYTAB_PRINCIPAL, remoteKeytabPrincipal);
    }
    ctx.setEnvironment(containerEnv);
    try (DataOutputBuffer dob = new DataOutputBuffer()) {
        log.debug("Adding security tokens to Task Executor Container launch Context....");
        UserGroupInformation user = UserGroupInformation.getCurrentUser();
        Credentials credentials = user.getCredentials();
        credentials.writeTokenStorageToStream(dob);
        ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        ctx.setTokens(securityTokens);
    } catch (Throwable t) {
        log.error("Getting current user info failed when trying to launch the container", t);
    }
    return ctx;
}

Example 78 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project hadoop by apache.

the class TestFSDownload method createTarFile.

static LocalResource createTarFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException, URISyntaxException {
    byte[] bytes = new byte[len];
    r.nextBytes(bytes);
    File archiveFile = new File(p.toUri().getPath() + ".tar");
    archiveFile.createNewFile();
    TarArchiveOutputStream out = new TarArchiveOutputStream(new FileOutputStream(archiveFile));
    TarArchiveEntry entry = new TarArchiveEntry(p.getName());
    entry.setSize(bytes.length);
    out.putArchiveEntry(entry);
    out.write(bytes);
    out.closeArchiveEntry();
    out.close();
    LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
    ret.setResource(URL.fromPath(new Path(p.toString() + ".tar")));
    ret.setSize(len);
    ret.setType(LocalResourceType.ARCHIVE);
    ret.setVisibility(vis);
    ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".tar")).getModificationTime());
    return ret;
}

Example 79 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project hadoop by apache.

the class TestFSDownload method createTgzFile.

static LocalResource createTgzFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException, URISyntaxException {
    byte[] bytes = new byte[len];
    r.nextBytes(bytes);
    File gzipFile = new File(p.toUri().getPath() + ".tar.gz");
    gzipFile.createNewFile();
    TarArchiveOutputStream out = new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(gzipFile)));
    TarArchiveEntry entry = new TarArchiveEntry(p.getName());
    entry.setSize(bytes.length);
    out.putArchiveEntry(entry);
    out.write(bytes);
    out.closeArchiveEntry();
    out.close();
    LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
    ret.setResource(URL.fromPath(new Path(p.toString() + ".tar.gz")));
    ret.setSize(len);
    ret.setType(LocalResourceType.ARCHIVE);
    ret.setVisibility(vis);
    ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".tar.gz")).getModificationTime());
    return ret;
}

Example 80 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project hadoop by apache.

the class TestFSDownload method testDownload.

@Test(timeout = 10000)
public void testDownload() throws IOException, URISyntaxException, InterruptedException {
    Configuration conf = new Configuration();
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
    FileContext files = FileContext.getLocalFSFileContext(conf);
    final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName()));
    files.mkdir(basedir, null, true);
    conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
    Map<LocalResource, LocalResourceVisibility> rsrcVis = new HashMap<LocalResource, LocalResourceVisibility>();
    Random rand = new Random();
    long sharedSeed = rand.nextLong();
    rand.setSeed(sharedSeed);
    System.out.println("SEED: " + sharedSeed);
    Map<LocalResource, Future<Path>> pending = new HashMap<LocalResource, Future<Path>>();
    ExecutorService exec = HadoopExecutors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(TestFSDownload.class.getName());
    int[] sizes = new int[10];
    for (int i = 0; i < 10; ++i) {
        sizes[i] = rand.nextInt(512) + 512;
        LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
        if (i % 2 == 1) {
            vis = LocalResourceVisibility.APPLICATION;
        }
        Path p = new Path(basedir, "" + i);
        LocalResource rsrc = createFile(files, p, sizes[i], rand, vis);
        rsrcVis.put(rsrc, vis);
        Path destPath = dirs.getLocalPathForWrite(basedir.toString(), sizes[i], conf);
        destPath = new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet()));
        FSDownload fsd = new FSDownload(files, UserGroupInformation.getCurrentUser(), conf, destPath, rsrc);
        pending.put(rsrc, exec.submit(fsd));
    }
    exec.shutdown();
    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
    for (Future<Path> path : pending.values()) {
        Assert.assertTrue(path.isDone());
    }
    try {
        for (Map.Entry<LocalResource, Future<Path>> p : pending.entrySet()) {
            Path localized = p.getValue().get();
            assertEquals(sizes[Integer.parseInt(localized.getName())], p.getKey().getSize());
            FileStatus status = files.getFileStatus(localized.getParent());
            FsPermission perm = status.getPermission();
            assertEquals("Cache directory permissions are incorrect", new FsPermission((short) 0755), perm);
            status = files.getFileStatus(localized);
            perm = status.getPermission();
            System.out.println("File permission " + perm + " for rsrc vis " + p.getKey().getVisibility().name());
            assert (rsrcVis.containsKey(p.getKey()));
            Assert.assertTrue("Private file should be 500", perm.toShort() == FSDownload.PRIVATE_FILE_PERMS.toShort());
        }
    } catch (ExecutionException e) {
        throw new IOException("Failed exec", e);
    }
}