Examples with AMPluginDescriptorProto org.apache.tez.dag.api.records.DAGProtos.AMPluginDescriptorProto used on opensource projects

Example 1 with AMPluginDescriptorProto

use of org.apache.tez.dag.api.records.DAGProtos.AMPluginDescriptorProto in project tez by apache.

the class DAGAppMaster method main.

public static void main(String[] args) {
    try {
        Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
        final String pid = System.getenv().get("JVM_PID");
        String containerIdStr = System.getenv(Environment.CONTAINER_ID.name());
        String nodeHostString = System.getenv(Environment.NM_HOST.name());
        String nodePortString = System.getenv(Environment.NM_PORT.name());
        String nodeHttpPortString = System.getenv(Environment.NM_HTTP_PORT.name());
        String appSubmitTimeStr = System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);
        String clientVersion = System.getenv(TezConstants.TEZ_CLIENT_VERSION_ENV);
        if (clientVersion == null) {
            clientVersion = VersionInfo.UNKNOWN;
        }
        validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV);
        ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
        ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId();
        long appSubmitTime = Long.parseLong(appSubmitTimeStr);
        String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name());
        // Command line options
        Options opts = new Options();
        opts.addOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION, false, "Run Tez Application Master in Session mode");
        CommandLine cliParser = new GnuParser().parse(opts, args);
        boolean sessionModeCliOption = cliParser.hasOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION);
        LOG.info("Creating DAGAppMaster for " + "applicationId=" + applicationAttemptId.getApplicationId() + ", attemptNum=" + applicationAttemptId.getAttemptId() + ", AMContainerId=" + containerId + ", jvmPid=" + pid + ", userFromEnv=" + jobUserName + ", cliSessionOption=" + sessionModeCliOption + ", pwd=" + System.getenv(Environment.PWD.name()) + ", localDirs=" + System.getenv(Environment.LOCAL_DIRS.name()) + ", logDirs=" + System.getenv(Environment.LOG_DIRS.name()));
        // TODO Does this really need to be a YarnConfiguration ?
        Configuration conf = new Configuration(new YarnConfiguration());
        ConfigurationProto confProto = TezUtilsInternal.readUserSpecifiedTezConfiguration(System.getenv(Environment.PWD.name()));
        TezUtilsInternal.addUserSpecifiedTezConfiguration(conf, confProto.getConfKeyValuesList());
        AMPluginDescriptorProto amPluginDescriptorProto = null;
        if (confProto.hasAmPluginDescriptor()) {
            amPluginDescriptorProto = confProto.getAmPluginDescriptor();
        }
        UserGroupInformation.setConfiguration(conf);
        Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
        TezUtilsInternal.setSecurityUtilConfigration(LOG, conf);
        DAGAppMaster appMaster = new DAGAppMaster(applicationAttemptId, containerId, nodeHostString, Integer.parseInt(nodePortString), Integer.parseInt(nodeHttpPortString), new SystemClock(), appSubmitTime, sessionModeCliOption, System.getenv(Environment.PWD.name()), TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOCAL_DIRS.name())), TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOG_DIRS.name())), clientVersion, credentials, jobUserName, amPluginDescriptorProto);
        ShutdownHookManager.get().addShutdownHook(new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
        // log the system properties
        if (LOG.isInfoEnabled()) {
            String systemPropsToLog = TezCommonUtils.getSystemPropertiesToLog(conf);
            if (systemPropsToLog != null) {
                LOG.info(systemPropsToLog);
            }
        }
        initAndStartAppMaster(appMaster, conf);
    } catch (Throwable t) {
        LOG.error("Error starting DAGAppMaster", t);
        System.exit(1);
    }
}

Example 2 with AMPluginDescriptorProto

use of org.apache.tez.dag.api.records.DAGProtos.AMPluginDescriptorProto in project tez by apache.

the class LocalClient method getPluginDescriptorInfo.

private AMPluginDescriptorProto getPluginDescriptorInfo(Configuration conf, String applicationIdString) throws IOException {
    Path tezSysStagingPath = TezCommonUtils.getTezSystemStagingPath(conf, applicationIdString);
    // Remove the filesystem qualifier.
    String unqualifiedPath = tezSysStagingPath.toUri().getPath();
    DAGProtos.ConfigurationProto confProto = TezUtilsInternal.readUserSpecifiedTezConfiguration(unqualifiedPath);
    AMPluginDescriptorProto amPluginDescriptorProto = null;
    if (confProto.hasAmPluginDescriptor()) {
        amPluginDescriptorProto = confProto.getAmPluginDescriptor();
    }
    return amPluginDescriptorProto;
}

Example 3 with AMPluginDescriptorProto

use of org.apache.tez.dag.api.records.DAGProtos.AMPluginDescriptorProto in project tez by apache.

the class TestDAGAppMaster method testParseAllPluginsCustomAndYarnSpecified.

@Test(timeout = 5000)
public void testParseAllPluginsCustomAndYarnSpecified() throws IOException {
    Configuration conf = new Configuration(false);
    conf.set(TEST_KEY, TEST_VAL);
    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
    TezUserPayloadProto payloadProto = TezUserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(defaultPayload.getPayload())).build();
    AMPluginDescriptorProto proto = createAmPluginDescriptor(true, false, true, payloadProto);
    List<NamedEntityDescriptor> tsDescriptors;
    BiMap<String, Integer> tsMap;
    List<NamedEntityDescriptor> clDescriptors;
    BiMap<String, Integer> clMap;
    List<NamedEntityDescriptor> tcDescriptors;
    BiMap<String, Integer> tcMap;
    // Only plugin, Yarn.
    tsDescriptors = Lists.newLinkedList();
    tsMap = HashBiMap.create();
    clDescriptors = Lists.newLinkedList();
    clMap = HashBiMap.create();
    tcDescriptors = Lists.newLinkedList();
    tcMap = HashBiMap.create();
    DAGAppMaster.parseAllPlugins(tsDescriptors, tsMap, clDescriptors, clMap, tcDescriptors, tcMap, proto, false, defaultPayload);
    verifyDescAndMap(tsDescriptors, tsMap, 2, true, TezConstants.getTezYarnServicePluginName(), TS_NAME);
    verifyDescAndMap(clDescriptors, clMap, 2, true, TezConstants.getTezYarnServicePluginName(), CL_NAME);
    verifyDescAndMap(tcDescriptors, tcMap, 2, true, TezConstants.getTezYarnServicePluginName(), TC_NAME);
    assertNull(tsDescriptors.get(0).getClassName());
    assertNull(clDescriptors.get(0).getClassName());
    assertNull(tcDescriptors.get(0).getClassName());
    assertEquals(TS_NAME + CLASS_SUFFIX, tsDescriptors.get(1).getClassName());
    assertEquals(CL_NAME + CLASS_SUFFIX, clDescriptors.get(1).getClassName());
    assertEquals(TC_NAME + CLASS_SUFFIX, tcDescriptors.get(1).getClassName());
}

Example 4 with AMPluginDescriptorProto

use of org.apache.tez.dag.api.records.DAGProtos.AMPluginDescriptorProto in project tez by apache.

the class TestDAGAppMaster method testParseAllPluginsOnlyCustomSpecified.

@Test(timeout = 5000)
public void testParseAllPluginsOnlyCustomSpecified() throws IOException {
    Configuration conf = new Configuration(false);
    conf.set(TEST_KEY, TEST_VAL);
    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
    TezUserPayloadProto payloadProto = TezUserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(defaultPayload.getPayload())).build();
    AMPluginDescriptorProto proto = createAmPluginDescriptor(false, false, true, payloadProto);
    List<NamedEntityDescriptor> tsDescriptors;
    BiMap<String, Integer> tsMap;
    List<NamedEntityDescriptor> clDescriptors;
    BiMap<String, Integer> clMap;
    List<NamedEntityDescriptor> tcDescriptors;
    BiMap<String, Integer> tcMap;
    // Only plugin, Yarn.
    tsDescriptors = Lists.newLinkedList();
    tsMap = HashBiMap.create();
    clDescriptors = Lists.newLinkedList();
    clMap = HashBiMap.create();
    tcDescriptors = Lists.newLinkedList();
    tcMap = HashBiMap.create();
    DAGAppMaster.parseAllPlugins(tsDescriptors, tsMap, clDescriptors, clMap, tcDescriptors, tcMap, proto, false, defaultPayload);
    verifyDescAndMap(tsDescriptors, tsMap, 2, true, TS_NAME, TezConstants.getTezYarnServicePluginName());
    verifyDescAndMap(clDescriptors, clMap, 1, true, CL_NAME);
    verifyDescAndMap(tcDescriptors, tcMap, 1, true, TC_NAME);
    assertEquals(TS_NAME + CLASS_SUFFIX, tsDescriptors.get(0).getClassName());
    assertEquals(CL_NAME + CLASS_SUFFIX, clDescriptors.get(0).getClassName());
    assertEquals(TC_NAME + CLASS_SUFFIX, tcDescriptors.get(0).getClassName());
}

Example 5 with AMPluginDescriptorProto

use of org.apache.tez.dag.api.records.DAGProtos.AMPluginDescriptorProto in project tez by apache.

the class TezClientUtils method createFinalConfProtoForApp.

static ConfigurationProto createFinalConfProtoForApp(Configuration amConf, ServicePluginsDescriptor servicePluginsDescriptor) {
    assert amConf != null;
    ConfigurationProto.Builder builder = ConfigurationProto.newBuilder();
    for (Entry<String, String> entry : amConf) {
        String key = entry.getKey();
        String val = amConf.get(key);
        if (val != null) {
            PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder();
            kvp.setKey(key);
            kvp.setValue(val);
            builder.addConfKeyValues(kvp);
        } else {
            LOG.debug("null value in Configuration after replacement for key={}. Skipping.", key);
        }
    }
    AMPluginDescriptorProto pluginDescriptorProto = DagTypeConverters.convertServicePluginDescriptorToProto(servicePluginsDescriptor);
    builder.setAmPluginDescriptor(pluginDescriptorProto);
    return builder.build();
}