Examples with VertexExecutionContext org.apache.tez.dag.api.Vertex.VertexExecutionContext used on opensource projects

Example 1 with VertexExecutionContext

use of org.apache.tez.dag.api.Vertex.VertexExecutionContext in project hive by apache.

the class DagUtils method createVertex.

/*
   * Helper function to create Vertex for given ReduceWork.
   */
private Vertex createVertex(JobConf conf, ReduceWork reduceWork, LocalResource appJarLr, List<LocalResource> additionalLr, FileSystem fs, Path mrScratchDir, Context ctx) throws Exception {
    // set up operator plan
    conf.set(Utilities.INPUT_NAME, reduceWork.getName());
    Utilities.setReduceWork(conf, reduceWork, mrScratchDir, false);
    // create the directories FileSinkOperators need
    Utilities.createTmpDirs(conf, reduceWork);
    VertexExecutionContext vertexExecutionContext = createVertexExecutionContext(reduceWork);
    // create the vertex
    Vertex reducer = Vertex.create(reduceWork.getName(), ProcessorDescriptor.create(ReduceTezProcessor.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(conf)), reduceWork.isAutoReduceParallelism() ? reduceWork.getMaxReduceTasks() : reduceWork.getNumReduceTasks(), getContainerResource(conf));
    reducer.setTaskEnvironment(getContainerEnvironment(conf, false));
    reducer.setExecutionContext(vertexExecutionContext);
    reducer.setTaskLaunchCmdOpts(getContainerJavaOpts(conf));
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(getBaseName(appJarLr), appJarLr);
    for (LocalResource lr : additionalLr) {
        localResources.put(getBaseName(lr), lr);
    }
    reducer.addTaskLocalFiles(localResources);
    return reducer;
}

Example 2 with VertexExecutionContext

use of org.apache.tez.dag.api.Vertex.VertexExecutionContext in project hive by apache.

the class DagUtils method createVertex.

/*
   * Helper function to create Vertex for given ReduceWork.
   */
private Vertex createVertex(JobConf conf, ReduceWork reduceWork, FileSystem fs, Path mrScratchDir, Context ctx, Map<String, LocalResource> localResources) throws Exception {
    // set up operator plan
    conf.set(Utilities.INPUT_NAME, reduceWork.getName());
    Utilities.setReduceWork(conf, reduceWork, mrScratchDir, false);
    // create the directories FileSinkOperators need
    Utilities.createTmpDirs(conf, reduceWork);
    VertexExecutionContext vertexExecutionContext = createVertexExecutionContext(reduceWork);
    // create the vertex
    Vertex reducer = Vertex.create(reduceWork.getName(), ProcessorDescriptor.create(ReduceTezProcessor.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(conf)), reduceWork.isAutoReduceParallelism() ? reduceWork.getMaxReduceTasks() : reduceWork.getNumReduceTasks(), getContainerResource(conf));
    reducer.setTaskEnvironment(getContainerEnvironment(conf, false));
    reducer.setExecutionContext(vertexExecutionContext);
    reducer.setTaskLaunchCmdOpts(getContainerJavaOpts(conf));
    reducer.addTaskLocalFiles(localResources);
    return reducer;
}

Example 3 with VertexExecutionContext

use of org.apache.tez.dag.api.Vertex.VertexExecutionContext in project tez by apache.

the class TestVertexImpl2 method testVertexExecutionContextOnly.

@Test(timeout = 5000)
public void testVertexExecutionContextOnly() {
    VertexExecutionContext vertexExecutionContext = VertexExecutionContext.create(ExecutionContextTestInfoHolder.append(ExecutionContextTestInfoHolder.TASK_SCHEDULER_NAME_BASE, 1), ExecutionContextTestInfoHolder.append(ExecutionContextTestInfoHolder.CONTAINER_LAUNCHER_NAME_BASE, 1), ExecutionContextTestInfoHolder.append(ExecutionContextTestInfoHolder.TASK_COMM_NAME_BASE, 1));
    ExecutionContextTestInfoHolder info = new ExecutionContextTestInfoHolder(vertexExecutionContext, null, 3);
    VertexWrapper vertexWrapper = createVertexWrapperForExecutionContextTest(info);
    assertEquals(1, vertexWrapper.vertex.taskSchedulerIdentifier);
    assertEquals(1, vertexWrapper.vertex.containerLauncherIdentifier);
    assertEquals(1, vertexWrapper.vertex.taskCommunicatorIdentifier);
}

Example 4 with VertexExecutionContext

use of org.apache.tez.dag.api.Vertex.VertexExecutionContext in project tez by apache.

the class TestDAGPlan method testInvalidExecContext_2.

@Test(timeout = 5000)
public void testInvalidExecContext_2() {
    ServicePluginsDescriptor servicePluginsDescriptor = ServicePluginsDescriptor.create(false, new TaskSchedulerDescriptor[] { TaskSchedulerDescriptor.create("plugin", null) }, new ContainerLauncherDescriptor[] { ContainerLauncherDescriptor.create("plugin", null) }, new TaskCommunicatorDescriptor[] { TaskCommunicatorDescriptor.create("plugin", null) });
    VertexExecutionContext validExecContext = VertexExecutionContext.create("plugin", "plugin", "plugin");
    VertexExecutionContext invalidExecContext1 = VertexExecutionContext.create("invalidplugin", "plugin", "plugin");
    VertexExecutionContext invalidExecContext2 = VertexExecutionContext.create("plugin", "invalidplugin", "plugin");
    VertexExecutionContext invalidExecContext3 = VertexExecutionContext.create("plugin", "plugin", "invalidplugin");
    DAG dag = DAG.create("dag1");
    dag.setExecutionContext(VertexExecutionContext.createExecuteInContainers(true));
    Vertex v1 = Vertex.create("testvertex", ProcessorDescriptor.create("processor1"), 1);
    dag.addVertex(v1);
    // Should succeed. Default context is containers.
    dag.createDag(new TezConfiguration(false), null, null, null, true, servicePluginsDescriptor, null);
    // Set execute in AM should fail
    v1.setExecutionContext(VertexExecutionContext.createExecuteInAm(true));
    try {
        dag.createDag(new TezConfiguration(false), null, null, null, true, servicePluginsDescriptor, null);
        fail("Expecting dag create to fail due to invalid ServicePluginDescriptor");
    } catch (IllegalStateException e) {
        assertTrue(e.getMessage().contains("AM execution"));
    }
    // Valid context
    v1.setExecutionContext(validExecContext);
    dag.createDag(new TezConfiguration(false), null, null, null, true, servicePluginsDescriptor, null);
    // Invalid task scheduler
    v1.setExecutionContext(invalidExecContext1);
    try {
        dag.createDag(new TezConfiguration(false), null, null, null, true, servicePluginsDescriptor, null);
        fail("Expecting dag create to fail due to invalid ServicePluginDescriptor");
    } catch (IllegalStateException e) {
        assertTrue(e.getMessage().contains("testvertex"));
        assertTrue(e.getMessage().contains("task scheduler"));
        assertTrue(e.getMessage().contains("invalidplugin"));
    }
    // Invalid ContainerLauncher
    v1.setExecutionContext(invalidExecContext2);
    try {
        dag.createDag(new TezConfiguration(false), null, null, null, true, servicePluginsDescriptor, null);
        fail("Expecting dag create to fail due to invalid ServicePluginDescriptor");
    } catch (IllegalStateException e) {
        assertTrue(e.getMessage().contains("testvertex"));
        assertTrue(e.getMessage().contains("container launcher"));
        assertTrue(e.getMessage().contains("invalidplugin"));
    }
    // Invalid task comm
    v1.setExecutionContext(invalidExecContext3);
    try {
        dag.createDag(new TezConfiguration(false), null, null, null, true, servicePluginsDescriptor, null);
        fail("Expecting dag create to fail due to invalid ServicePluginDescriptor");
    } catch (IllegalStateException e) {
        assertTrue(e.getMessage().contains("testvertex"));
        assertTrue(e.getMessage().contains("task communicator"));
        assertTrue(e.getMessage().contains("invalidplugin"));
    }
}

Example 5 with VertexExecutionContext

use of org.apache.tez.dag.api.Vertex.VertexExecutionContext in project tez by apache.

the class TestDagTypeConverters method testVertexExecutionContextTranslation.

@Test(timeout = 5000)
public void testVertexExecutionContextTranslation() {
    VertexExecutionContext originalContext;
    VertexExecutionContextProto contextProto;
    VertexExecutionContext retrievedContext;
    // Uber
    originalContext = VertexExecutionContext.createExecuteInAm(true);
    contextProto = DagTypeConverters.convertToProto(originalContext);
    retrievedContext = DagTypeConverters.convertFromProto(contextProto);
    assertEquals(originalContext, retrievedContext);
    // Regular containers
    originalContext = VertexExecutionContext.createExecuteInContainers(true);
    contextProto = DagTypeConverters.convertToProto(originalContext);
    retrievedContext = DagTypeConverters.convertFromProto(contextProto);
    assertEquals(originalContext, retrievedContext);
    // Custom
    originalContext = VertexExecutionContext.create("plugin", "plugin", "plugin");
    contextProto = DagTypeConverters.convertToProto(originalContext);
    retrievedContext = DagTypeConverters.convertFromProto(contextProto);
    assertEquals(originalContext, retrievedContext);
}