Examples with LocalResource org.apache.hadoop.yarn.api.records.LocalResource used on opensource projects

Example 66 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project hive by apache.

the class TestTezTask method testExtraResourcesAddedToDag.

@Test
public void testExtraResourcesAddedToDag() throws Exception {
    final String[] inputOutputJars = new String[] { "file:///tmp/foo.jar" };
    LocalResource res = mock(LocalResource.class);
    final List<LocalResource> resources = Collections.singletonList(res);
    final Map<String, LocalResource> resMap = new HashMap<String, LocalResource>();
    resMap.put("foo.jar", res);
    DAG dag = mock(DAG.class);
    when(utils.localizeTempFiles(path.toString(), conf, inputOutputJars)).thenReturn(resources);
    when(utils.getBaseName(res)).thenReturn("foo.jar");
    when(sessionState.isOpen()).thenReturn(true);
    when(sessionState.isOpening()).thenReturn(false);
    when(sessionState.hasResources(inputOutputJars)).thenReturn(false);
    task.addExtraResourcesToDag(sessionState, dag, inputOutputJars, resMap);
    verify(dag).addTaskLocalFiles(resMap);
}

Example 67 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project hive by apache.

the class TezTask method getExtraLocalResources.

/**
   * Converted the list of jars into local resources
   */
Map<String, LocalResource> getExtraLocalResources(JobConf jobConf, Path scratchDir, String[] inputOutputJars) throws Exception {
    final Map<String, LocalResource> resources = new HashMap<String, LocalResource>();
    final List<LocalResource> localResources = utils.localizeTempFiles(scratchDir.toString(), jobConf, inputOutputJars);
    if (null != localResources) {
        for (LocalResource lr : localResources) {
            resources.put(utils.getBaseName(lr), lr);
        }
    }
    return resources;
}

Example 68 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project kitten by cloudera.

the class LuaContainerLaunchParameters method getLocalResources.

@Override
public Map<String, LocalResource> getLocalResources() {
    Map<String, LocalResource> localResources = Maps.newHashMap();
    if (!lv.isNil(LuaFields.RESOURCES)) {
        LuaWrapper lr = lv.getTable(LuaFields.RESOURCES);
        for (LuaPair lp : lr) {
            try {
                NamedLocalResource nlr = constructResource(lp);
                localResources.put(nlr.name, nlr.resource);
            } catch (IOException e) {
                LOG.error("Error constructing local resource: " + lp.key, e);
            }
        }
    }
    for (Map.Entry<String, String> elr : extras.getResources().entrySet()) {
        LocalResource rsrc = constructExtraResource(elr.getValue());
        if (rsrc != null) {
            localResources.put(elr.getKey(), rsrc);
        }
    }
    // Get a local resource for the configuration object.
    LocalResource confRsrc = constructExtraResource(LuaFields.KITTEN_JOB_XML_FILE);
    if (confRsrc != null) {
        localResources.put(LuaFields.KITTEN_JOB_XML_FILE, confRsrc);
    }
    return localResources;
}

Example 69 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project kitten by cloudera.

the class LuaContainerLaunchParameters method constructResource.

private NamedLocalResource constructResource(LuaPair lp) throws IOException {
    LocalResource rsrc = Records.newRecord(LocalResource.class);
    LuaWrapper value = new LuaWrapper(lp.value.checktable());
    String name = lp.key.isint() ? "" : lp.key.tojstring();
    if (value.isNil(LuaFields.LOCAL_RESOURCE_TYPE)) {
        rsrc.setType(LocalResourceType.FILE);
    } else {
        rsrc.setType(LocalResourceType.valueOf(value.getString(LuaFields.LOCAL_RESOURCE_TYPE).toUpperCase()));
    }
    if (value.isNil(LuaFields.LOCAL_RESOURCE_VISIBILITY)) {
        rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    } else {
        rsrc.setVisibility(LocalResourceVisibility.valueOf(value.getString(LuaFields.LOCAL_RESOURCE_VISIBILITY).toUpperCase()));
    }
    if (!value.isNil(LuaFields.LOCAL_RESOURCE_URL)) {
        URI uri = URI.create(value.getString(LuaFields.LOCAL_RESOURCE_URL));
        rsrc.setResource(ConverterUtils.getYarnUrlFromURI(uri));
        if (name.isEmpty()) {
            name = (new File(uri.getPath())).getName();
        }
    } else if (!value.isNil(LuaFields.LOCAL_RESOURCE_HDFS_FILE)) {
        Path path = new Path(value.getString(LuaFields.LOCAL_RESOURCE_HDFS_FILE));
        configureLocalResourceForPath(rsrc, path);
        if (name.isEmpty()) {
            name = path.getName();
        }
    } else if (!value.isNil(LuaFields.LOCAL_RESOURCE_LOCAL_FILE)) {
        String src = value.getString(LuaFields.LOCAL_RESOURCE_LOCAL_FILE);
        Path path = new Path(localFileUris.get(src));
        configureLocalResourceForPath(rsrc, path);
        if (name.isEmpty()) {
            name = new Path(src).getName();
        }
    } else {
        throw new IllegalArgumentException("Invalid resource: no 'url', 'hdfs', or 'file' fields specified.");
    }
    return new NamedLocalResource(name, rsrc);
}

Example 70 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project jstorm by alibaba.

the class JstormOnYarn method addToLocalResources.

private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map<String, LocalResource> localResources, String resources) throws IOException {
    String suffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId + JOYConstants.BACKLASH + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {
            ostream = FileSystem.create(fs, dst, new FsPermission(JOYConstants.FS_PERMISSION));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}