Examples with ConfigurationProto org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto used on opensource projects

Example 1 with ConfigurationProto

use of org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto in project tez by apache.

the class DAGAppMaster method main.

public static void main(String[] args) {
    try {
        Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
        final String pid = System.getenv().get("JVM_PID");
        String containerIdStr = System.getenv(Environment.CONTAINER_ID.name());
        String nodeHostString = System.getenv(Environment.NM_HOST.name());
        String nodePortString = System.getenv(Environment.NM_PORT.name());
        String nodeHttpPortString = System.getenv(Environment.NM_HTTP_PORT.name());
        String appSubmitTimeStr = System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);
        String clientVersion = System.getenv(TezConstants.TEZ_CLIENT_VERSION_ENV);
        if (clientVersion == null) {
            clientVersion = VersionInfo.UNKNOWN;
        }
        validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV);
        ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
        ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId();
        long appSubmitTime = Long.parseLong(appSubmitTimeStr);
        String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name());
        // Command line options
        Options opts = new Options();
        opts.addOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION, false, "Run Tez Application Master in Session mode");
        CommandLine cliParser = new GnuParser().parse(opts, args);
        boolean sessionModeCliOption = cliParser.hasOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION);
        LOG.info("Creating DAGAppMaster for " + "applicationId=" + applicationAttemptId.getApplicationId() + ", attemptNum=" + applicationAttemptId.getAttemptId() + ", AMContainerId=" + containerId + ", jvmPid=" + pid + ", userFromEnv=" + jobUserName + ", cliSessionOption=" + sessionModeCliOption + ", pwd=" + System.getenv(Environment.PWD.name()) + ", localDirs=" + System.getenv(Environment.LOCAL_DIRS.name()) + ", logDirs=" + System.getenv(Environment.LOG_DIRS.name()));
        // TODO Does this really need to be a YarnConfiguration ?
        Configuration conf = new Configuration(new YarnConfiguration());
        ConfigurationProto confProto = TezUtilsInternal.readUserSpecifiedTezConfiguration(System.getenv(Environment.PWD.name()));
        TezUtilsInternal.addUserSpecifiedTezConfiguration(conf, confProto.getConfKeyValuesList());
        AMPluginDescriptorProto amPluginDescriptorProto = null;
        if (confProto.hasAmPluginDescriptor()) {
            amPluginDescriptorProto = confProto.getAmPluginDescriptor();
        }
        UserGroupInformation.setConfiguration(conf);
        Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
        TezUtilsInternal.setSecurityUtilConfigration(LOG, conf);
        DAGAppMaster appMaster = new DAGAppMaster(applicationAttemptId, containerId, nodeHostString, Integer.parseInt(nodePortString), Integer.parseInt(nodeHttpPortString), new SystemClock(), appSubmitTime, sessionModeCliOption, System.getenv(Environment.PWD.name()), TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOCAL_DIRS.name())), TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOG_DIRS.name())), clientVersion, credentials, jobUserName, amPluginDescriptorProto);
        ShutdownHookManager.get().addShutdownHook(new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
        // log the system properties
        if (LOG.isInfoEnabled()) {
            String systemPropsToLog = TezCommonUtils.getSystemPropertiesToLog(conf);
            if (systemPropsToLog != null) {
                LOG.info(systemPropsToLog);
            }
        }
        initAndStartAppMaster(appMaster, conf);
    } catch (Throwable t) {
        LOG.error("Error starting DAGAppMaster", t);
        System.exit(1);
    }
}

Example 2 with ConfigurationProto

use of org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto in project tez by apache.

the class TestTezClientUtils method testConfSerializationForAm.

@Test(timeout = 5000)
public void testConfSerializationForAm() {
    Configuration conf = new Configuration(false);
    String val1 = "fixedProperty";
    String val2 = "parametrizedProperty/${user.name}";
    String expVal2 = "parametrizedProperty/" + System.getProperty("user.name");
    conf.set("property1", val1);
    conf.set("property2", val2);
    Map<String, String> expected = new HashMap<String, String>();
    expected.put("property1", val1);
    expected.put("property2", expVal2);
    ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(conf, null);
    for (PlanKeyValuePair kvPair : confProto.getConfKeyValuesList()) {
        String v = expected.remove(kvPair.getKey());
        assertEquals(v, kvPair.getValue());
    }
    assertTrue(expected.isEmpty());
}

Example 3 with ConfigurationProto

use of org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto in project tez by apache.

the class TestTezClientUtils method testConfigurationAllowAll.

@Test(timeout = 5000)
public void testConfigurationAllowAll() {
    Configuration srcConf = new Configuration(false);
    Map<String, String> confMap = new HashMap<String, String>();
    confMap.put("foo.property", "2000");
    confMap.put("tez.property", "tezProperty");
    confMap.put("yarn.property", "yarnProperty");
    for (Map.Entry<String, String> entry : confMap.entrySet()) {
        srcConf.set(entry.getKey(), entry.getValue());
    }
    ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(srcConf, null);
    for (PlanKeyValuePair kvPair : confProto.getConfKeyValuesList()) {
        String val = confMap.remove(kvPair.getKey());
        assertNotNull(val);
        assertEquals(val, kvPair.getValue());
    }
    assertTrue(confMap.isEmpty());
}

Example 4 with ConfigurationProto

use of org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto in project tez by apache.

the class TestTezClientUtils method testServiceDescriptorSerializationForAM.

@Test(timeout = 5000)
public void testServiceDescriptorSerializationForAM() {
    Configuration conf = new Configuration(false);
    ServicePluginsDescriptor servicePluginsDescriptor = ServicePluginsDescriptor.create(true);
    ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(conf, servicePluginsDescriptor);
    assertTrue(confProto.hasAmPluginDescriptor());
    assertTrue(confProto.getAmPluginDescriptor().getUberEnabled());
}

Example 5 with ConfigurationProto

use of org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto in project tez by apache.

the class TestVertexImpl2 method createVertexPlanForExeuctionContextTests.

private VertexPlan createVertexPlanForExeuctionContextTests(ExecutionContextTestInfoHolder info) {
    ConfigurationProto confProto = ConfigurationProto.newBuilder().addConfKeyValues(PlanKeyValuePair.newBuilder().setKey("foo").setValue("bar").build()).addConfKeyValues(PlanKeyValuePair.newBuilder().setKey("foo1").setValue("bar2").build()).build();
    VertexPlan.Builder vertexPlanBuilder = VertexPlan.newBuilder().setName(info.vertexName).setVertexConf(confProto).setTaskConfig(DAGProtos.PlanTaskConfiguration.newBuilder().setNumTasks(10).setJavaOpts("dontcare").setMemoryMb(1024).setVirtualCores(1).setTaskModule("taskmodule").build()).setType(DAGProtos.PlanVertexType.NORMAL);
    if (info.vertexExecutionContext != null) {
        vertexPlanBuilder.setExecutionContext(DagTypeConverters.convertToProto(info.vertexExecutionContext));
    }
    return vertexPlanBuilder.build();
}