Examples with Private org.apache.hadoop.classification.InterfaceAudience.Private used on opensource projects

Example 66 with Private

use of org.apache.hadoop.classification.InterfaceAudience.Private in project tez by apache.

the class ReflectionUtils method addResourcesToSystemClassLoader.

@Private
public static synchronized void addResourcesToSystemClassLoader(List<URL> urls) {
    URLClassLoader sysLoader = (URLClassLoader) ClassLoader.getSystemClassLoader();
    if (sysClassLoaderMethod == null) {
        Class<?> sysClass = URLClassLoader.class;
        Method method;
        try {
            method = sysClass.getDeclaredMethod("addURL", parameters);
        } catch (SecurityException e) {
            throw new TezUncheckedException("Failed to get handle on method addURL", e);
        } catch (NoSuchMethodException e) {
            throw new TezUncheckedException("Failed to get handle on method addURL", e);
        }
        method.setAccessible(true);
        sysClassLoaderMethod = method;
    }
    for (URL url : urls) {
        try {
            sysClassLoaderMethod.invoke(sysLoader, new Object[] { url });
        } catch (IllegalArgumentException e) {
            throw new TezUncheckedException("Failed to invoke addURL for rsrc: " + url, e);
        } catch (IllegalAccessException e) {
            throw new TezUncheckedException("Failed to invoke addURL for rsrc: " + url, e);
        } catch (InvocationTargetException e) {
            throw new TezUncheckedException("Failed to invoke addURL for rsrc: " + url, e);
        }
    }
}

Example 67 with Private

use of org.apache.hadoop.classification.InterfaceAudience.Private in project tez by apache.

the class ReflectionUtils method addResourcesToClasspath.

@Private
public static synchronized void addResourcesToClasspath(List<URL> urls) {
    ClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), Thread.currentThread().getContextClassLoader());
    Thread.currentThread().setContextClassLoader(classLoader);
}

Example 68 with Private

use of org.apache.hadoop.classification.InterfaceAudience.Private in project tez by apache.

the class TezCommonUtils method getTezSystemStagingPath.

/**
 * <p>
 * This function returns the staging directory for TEZ system. Tez creates all
 * its temporary files under this sub-directory. The function normally doesn't
 * creates any sub-directory under the base staging directory.
 * </p>
 *
 * @param conf
 *          Tez configuration
 * @param strAppId
 *          Application ID as string
 * @return TEZ system level staging directory used for Tez internals
 */
@Private
public static Path getTezSystemStagingPath(Configuration conf, String strAppId) {
    Path baseStagingPath = getTezBaseStagingPath(conf);
    Path tezStagingDir;
    tezStagingDir = new Path(baseStagingPath, TEZ_SYSTEM_SUB_DIR);
    tezStagingDir = new Path(tezStagingDir, strAppId);
    return tezStagingDir;
}

Example 69 with Private

use of org.apache.hadoop.classification.InterfaceAudience.Private in project tez by apache.

the class TezClient method submitDAGApplication.

// To be used only by YarnRunner
@Private
DAGClient submitDAGApplication(ApplicationId appId, DAG dag) throws TezException, IOException {
    LOG.info("Submitting DAG application with id: " + appId);
    try {
        // Use the AMCredentials object in client mode, since this won't be re-used.
        // Ensures we don't fetch credentially unnecessarily if the user has already provided them.
        Credentials credentials = amConfig.getCredentials();
        if (credentials == null) {
            credentials = new Credentials();
        }
        TezClientUtils.processTezLocalCredentialsFile(credentials, amConfig.getTezConfiguration());
        // Add session token for shuffle
        TezClientUtils.createSessionToken(appId.toString(), jobTokenSecretManager, credentials);
        // Add credentials for tez-local resources.
        Map<String, LocalResource> tezJarResources = getTezJarResources(credentials);
        ApplicationSubmissionContext appContext = TezClientUtils.createApplicationSubmissionContext(appId, dag, dag.getName(), amConfig, tezJarResources, credentials, usingTezArchiveDeploy, apiVersionInfo, servicePluginsDescriptor, javaOptsChecker);
        String callerContextStr = "";
        if (dag.getCallerContext() != null) {
            callerContextStr = ", callerContext=" + dag.getCallerContext().contextAsSimpleString();
        }
        LOG.info("Submitting DAG to YARN" + ", applicationId=" + appId + ", dagName=" + dag.getName() + callerContextStr);
        frameworkClient.submitApplication(appContext);
        ApplicationReport appReport = frameworkClient.getApplicationReport(appId);
        LOG.info("The url to track the Tez AM: " + appReport.getTrackingUrl());
        lastSubmittedAppId = appId;
    } catch (YarnException e) {
        throw new TezException(e);
    }
    // wait for dag in non-session mode to start running, so that we can start to getDAGStatus
    waitNonSessionTillReady();
    return getDAGClient(appId, amConfig.getTezConfiguration(), amConfig.getYarnConfiguration(), frameworkClient);
}

Example 70 with Private

use of org.apache.hadoop.classification.InterfaceAudience.Private in project tez by apache.

the class TezCommonUtils method compressByteArrayToByteString.

@Private
public static ByteString compressByteArrayToByteString(byte[] inBytes, Deflater deflater) throws IOException {
    deflater.reset();
    ByteString.Output os = ByteString.newOutput();
    DeflaterOutputStream compressOs = null;
    try {
        compressOs = new DeflaterOutputStream(os, deflater);
        compressOs.write(inBytes);
        compressOs.finish();
        ByteString byteString = os.toByteString();
        return byteString;
    } finally {
        if (compressOs != null) {
            compressOs.close();
        }
    }
}