Examples with ProcessorDescriptor org.apache.tez.dag.api.ProcessorDescriptor used on opensource projects

Example 11 with ProcessorDescriptor

use of org.apache.tez.dag.api.ProcessorDescriptor in project hive by apache.

the class TestConverters method testTaskSpecToFragmentSpec.

@Test(timeout = 10000)
public void testTaskSpecToFragmentSpec() {
    ByteBuffer procBb = ByteBuffer.allocate(4);
    procBb.putInt(0, 200);
    UserPayload processorPayload = UserPayload.create(procBb);
    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create("fakeProcessorName").setUserPayload(processorPayload);
    ByteBuffer input1Bb = ByteBuffer.allocate(4);
    input1Bb.putInt(0, 300);
    UserPayload input1Payload = UserPayload.create(input1Bb);
    InputDescriptor id1 = InputDescriptor.create("input1ClassName").setUserPayload(input1Payload);
    InputSpec inputSpec1 = new InputSpec("sourceVertexName1", id1, 33);
    InputSpec inputSpec2 = new InputSpec("sourceVertexName2", id1, 44);
    List<InputSpec> inputSpecList = Lists.newArrayList(inputSpec1, inputSpec2);
    ByteBuffer output1Bb = ByteBuffer.allocate(4);
    output1Bb.putInt(0, 400);
    UserPayload output1Payload = UserPayload.create(output1Bb);
    OutputDescriptor od1 = OutputDescriptor.create("output1ClassName").setUserPayload(output1Payload);
    OutputSpec outputSpec1 = new OutputSpec("destVertexName1", od1, 55);
    OutputSpec outputSpec2 = new OutputSpec("destVertexName2", od1, 66);
    List<OutputSpec> outputSpecList = Lists.newArrayList(outputSpec1, outputSpec2);
    ApplicationId appId = ApplicationId.newInstance(1000, 100);
    TezDAGID tezDagId = TezDAGID.getInstance(appId, 300);
    TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400);
    TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500);
    TezTaskAttemptID tezTaskAttemptId = TezTaskAttemptID.getInstance(tezTaskId, 600);
    TaskSpec taskSpec = new TaskSpec(tezTaskAttemptId, "dagName", "vertexName", 10, processorDescriptor, inputSpecList, outputSpecList, null);
    QueryIdentifierProto queryIdentifierProto = QueryIdentifierProto.newBuilder().setApplicationIdString(appId.toString()).setAppAttemptNumber(333).setDagIndex(300).build();
    SignableVertexSpec vertexProto = Converters.constructSignableVertexSpec(taskSpec, queryIdentifierProto, "", "", "hiveQueryId").build();
    assertEquals("dagName", vertexProto.getDagName());
    assertEquals("vertexName", vertexProto.getVertexName());
    assertEquals("hiveQueryId", vertexProto.getHiveQueryId());
    assertEquals(appId.toString(), vertexProto.getQueryIdentifier().getApplicationIdString());
    assertEquals(tezDagId.getId(), vertexProto.getQueryIdentifier().getDagIndex());
    assertEquals(333, vertexProto.getQueryIdentifier().getAppAttemptNumber());
    assertEquals(tezVertexId.getId(), vertexProto.getVertexIndex());
    assertEquals(processorDescriptor.getClassName(), vertexProto.getProcessorDescriptor().getClassName());
    assertEquals(processorDescriptor.getUserPayload().getPayload(), vertexProto.getProcessorDescriptor().getUserPayload().getUserPayload().asReadOnlyByteBuffer());
    assertEquals(2, vertexProto.getInputSpecsCount());
    assertEquals(2, vertexProto.getOutputSpecsCount());
    verifyInputSpecAndProto(inputSpec1, vertexProto.getInputSpecs(0));
    verifyInputSpecAndProto(inputSpec2, vertexProto.getInputSpecs(1));
    verifyOutputSpecAndProto(outputSpec1, vertexProto.getOutputSpecs(0));
    verifyOutputSpecAndProto(outputSpec2, vertexProto.getOutputSpecs(1));
}

Example 12 with ProcessorDescriptor

use of org.apache.tez.dag.api.ProcessorDescriptor in project tez by apache.

the class TestVertexImpl method testFailuresMaxPercentExceededSourceTaskAttemptCompletionEvents.

@Test(timeout = 5000)
public void testFailuresMaxPercentExceededSourceTaskAttemptCompletionEvents() throws TezException {
    LOG.info("Testing testFailuresMaxPercentSourceTaskAttemptCompletionEvents");
    // Override the basic setup for this test to inject the specific config setting needed for this test
    useCustomInitializer = false;
    customInitializer = null;
    setupPreDagCreation();
    conf.setFloat(TezConfiguration.TEZ_VERTEX_FAILURES_MAXPERCENT, 50.0f);
    conf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1);
    dagPlan = createTestDAGPlan();
    setupPostDagCreation();
    initAllVertices(VertexState.INITED);
    VertexImpl v4 = vertices.get("vertex4");
    VertexImpl v5 = vertices.get("vertex5");
    VertexImpl v6 = vertices.get("vertex6");
    startVertex(vertices.get("vertex1"));
    startVertex(vertices.get("vertex2"));
    dispatcher.await();
    LOG.info("Verifying v6 state " + v6.getState());
    Assert.assertEquals(VertexState.RUNNING, v6.getState());
    TezTaskID t1_v4 = TezTaskID.getInstance(v4.getVertexId(), 0);
    TezTaskID t2_v4 = TezTaskID.getInstance(v4.getVertexId(), 1);
    TezTaskID t1_v5 = TezTaskID.getInstance(v5.getVertexId(), 0);
    TezTaskID t2_v5 = TezTaskID.getInstance(v5.getVertexId(), 1);
    TezTaskAttemptID ta1_t1_v4 = TezTaskAttemptID.getInstance(t1_v4, 0);
    TezTaskAttemptID ta1_t2_v4 = TezTaskAttemptID.getInstance(t2_v4, 0);
    TezTaskAttemptID ta1_t1_v5 = TezTaskAttemptID.getInstance(t1_v5, 0);
    TezTaskAttemptID ta1_t2_v5 = TezTaskAttemptID.getInstance(t2_v5, 0);
    TaskSpec taskSpec = new TaskSpec("dag", "vertex", 2, new ProcessorDescriptor(), new ArrayList<InputSpec>(), new ArrayList<OutputSpec>(), null, conf);
    TaskLocationHint locationHint = TaskLocationHint.createTaskLocationHint(null, null);
    // Tasks can only succeed from a scheduled or running state
    dispatcher.getEventHandler().handle(new TaskEventScheduleTask(t1_v4, taskSpec, locationHint, false));
    dispatcher.getEventHandler().handle(new TaskEventScheduleTask(t2_v4, taskSpec, locationHint, false));
    // Completed tasks are more that the max percent failure
    dispatcher.getEventHandler().handle(new TaskEventTAFailed(ta1_t1_v4, TaskFailureType.NON_FATAL, null));
    dispatcher.getEventHandler().handle(new TaskEventTAFailed(ta1_t2_v4, TaskFailureType.NON_FATAL, null));
    dispatcher.getEventHandler().handle(new TaskEventTASucceeded(ta1_t1_v5));
    dispatcher.getEventHandler().handle(new TaskEventTAFailed(ta1_t2_v5, TaskFailureType.NON_FATAL, null));
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v4.getState());
    Assert.assertEquals(VertexState.SUCCEEDED, v5.getState());
    Assert.assertEquals(VertexState.RUNNING, v6.getState());
    Assert.assertEquals(2, v6.numSuccessSourceAttemptCompletions);
}

Example 13 with ProcessorDescriptor

use of org.apache.tez.dag.api.ProcessorDescriptor in project tez by apache.

the class ProtoConverters method getTaskSpecfromProto.

public static TaskSpec getTaskSpecfromProto(TaskSpecProto taskSpecProto) {
    TezTaskAttemptID taskAttemptID = TezTaskAttemptID.fromString(taskSpecProto.getTaskAttemptIdString());
    ProcessorDescriptor processorDescriptor = null;
    if (taskSpecProto.hasProcessorDescriptor()) {
        processorDescriptor = DagTypeConverters.convertProcessorDescriptorFromDAGPlan(taskSpecProto.getProcessorDescriptor());
    }
    List<InputSpec> inputSpecList = new ArrayList<InputSpec>(taskSpecProto.getInputSpecsCount());
    if (taskSpecProto.getInputSpecsCount() > 0) {
        for (IOSpecProto inputSpecProto : taskSpecProto.getInputSpecsList()) {
            inputSpecList.add(getInputSpecFromProto(inputSpecProto));
        }
    }
    List<OutputSpec> outputSpecList = new ArrayList<OutputSpec>(taskSpecProto.getOutputSpecsCount());
    if (taskSpecProto.getOutputSpecsCount() > 0) {
        for (IOSpecProto outputSpecProto : taskSpecProto.getOutputSpecsList()) {
            outputSpecList.add(getOutputSpecFromProto(outputSpecProto));
        }
    }
    List<GroupInputSpec> groupInputSpecs = new ArrayList<GroupInputSpec>(taskSpecProto.getGroupedInputSpecsCount());
    if (taskSpecProto.getGroupedInputSpecsCount() > 0) {
        for (GroupInputSpecProto groupInputSpecProto : taskSpecProto.getGroupedInputSpecsList()) {
            groupInputSpecs.add(getGroupInputSpecFromProto(groupInputSpecProto));
        }
    }
    Configuration taskConf = null;
    if (taskSpecProto.hasTaskConf()) {
        taskConf = new Configuration(false);
        Map<String, String> confMap = DagTypeConverters.convertConfFromProto(taskSpecProto.getTaskConf());
        for (Entry<String, String> e : confMap.entrySet()) {
            taskConf.set(e.getKey(), e.getValue());
        }
    }
    TaskSpec taskSpec = new TaskSpec(taskAttemptID, taskSpecProto.getDagName(), taskSpecProto.getVertexName(), taskSpecProto.getVertexParallelism(), processorDescriptor, inputSpecList, outputSpecList, groupInputSpecs, taskConf);
    return taskSpec;
}

Example 14 with ProcessorDescriptor

use of org.apache.tez.dag.api.ProcessorDescriptor in project tez by apache.

the class TestTezClient method _testTezClientSessionLargeDAGPlan.

private void _testTezClientSessionLargeDAGPlan(int maxIPCMsgSize, int payloadSize, int amResourceSize, boolean shouldSerialize) throws Exception {
    TezConfiguration conf = new TezConfiguration();
    conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, maxIPCMsgSize);
    conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, "target/" + this.getClass().getName());
    TezClientForTest client = configureAndCreateTezClient(null, true, conf);
    Map<String, LocalResource> localResourceMap = new HashMap<>();
    byte[] bytes = new byte[amResourceSize];
    Arrays.fill(bytes, (byte) 1);
    String lrName = new String(bytes);
    localResourceMap.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create("P");
    processorDescriptor.setUserPayload(UserPayload.create(ByteBuffer.allocate(payloadSize)));
    Vertex vertex = Vertex.create("Vertex", processorDescriptor, 1, Resource.newInstance(1, 1));
    DAG dag = DAG.create("DAG").addVertex(vertex);
    client.start();
    client.addAppMasterLocalFiles(localResourceMap);
    client.submitDAG(dag);
    client.stop();
    ArgumentCaptor<SubmitDAGRequestProto> captor = ArgumentCaptor.forClass(SubmitDAGRequestProto.class);
    verify(client.sessionAmProxy).submitDAG((RpcController) any(), captor.capture());
    SubmitDAGRequestProto request = captor.getValue();
    if (shouldSerialize) {
        /* we need manually delete the serialized dagplan since staging path here won't be destroyed */
        Path dagPlanPath = new Path(request.getSerializedRequestPath());
        FileSystem fs = FileSystem.getLocal(conf);
        fs.deleteOnExit(dagPlanPath);
        fs.delete(dagPlanPath, false);
        assertTrue(request.hasSerializedRequestPath());
        assertFalse(request.hasDAGPlan());
        assertFalse(request.hasAdditionalAmResources());
    } else {
        assertFalse(request.hasSerializedRequestPath());
        assertTrue(request.hasDAGPlan());
        assertTrue(request.hasAdditionalAmResources());
    }
}

Example 15 with ProcessorDescriptor

use of org.apache.tez.dag.api.ProcessorDescriptor in project tez by apache.

the class TestProcessorContext method testDagNumber.

@Test(timeout = 5000)
public void testDagNumber() throws IOException {
    String[] localDirs = new String[] { "dummyLocalDir" };
    int appAttemptNumber = 1;
    TezUmbilical tezUmbilical = mock(TezUmbilical.class);
    String dagName = "DAG_NAME";
    String vertexName = "VERTEX_NAME";
    int vertexParallelism = 20;
    int dagNumber = 52;
    ApplicationId appId = ApplicationId.newInstance(10000, 13);
    TezDAGID dagId = TezDAGID.getInstance(appId, dagNumber);
    TezVertexID vertexId = TezVertexID.getInstance(dagId, 6);
    TezTaskID taskId = TezTaskID.getInstance(vertexId, 4);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 2);
    TaskSpec mockSpec = mock(TaskSpec.class);
    when(mockSpec.getInputs()).thenReturn(Collections.singletonList(mock(InputSpec.class)));
    when(mockSpec.getOutputs()).thenReturn(Collections.singletonList(mock(OutputSpec.class)));
    Configuration conf = new Configuration();
    TezSharedExecutor sharedExecutor = new TezSharedExecutor(conf);
    LogicalIOProcessorRuntimeTask runtimeTask = new LogicalIOProcessorRuntimeTask(mockSpec, 1, conf, new String[] { "/" }, tezUmbilical, null, null, null, null, "", null, 1024, false, new DefaultHadoopShim(), sharedExecutor);
    LogicalIOProcessorRuntimeTask mockTask = spy(runtimeTask);
    Map<String, ByteBuffer> serviceConsumerMetadata = Maps.newHashMap();
    Map<String, String> auxServiceEnv = Maps.newHashMap();
    MemoryDistributor memDist = mock(MemoryDistributor.class);
    ProcessorDescriptor processorDesc = mock(ProcessorDescriptor.class);
    InputReadyTracker inputReadyTracker = mock(InputReadyTracker.class);
    ObjectRegistry objectRegistry = new ObjectRegistryImpl();
    ExecutionContext execContext = new ExecutionContextImpl("localhost");
    long memAvailable = 10000l;
    TezProcessorContextImpl procContext = new TezProcessorContextImpl(new Configuration(), localDirs, appAttemptNumber, tezUmbilical, dagName, vertexName, vertexParallelism, taskAttemptId, null, runtimeTask, serviceConsumerMetadata, auxServiceEnv, memDist, processorDesc, inputReadyTracker, objectRegistry, execContext, memAvailable, sharedExecutor);
    assertEquals(dagNumber, procContext.getDagIdentifier());
    assertEquals(appAttemptNumber, procContext.getDAGAttemptNumber());
    assertEquals(appId, procContext.getApplicationId());
    assertEquals(dagName, procContext.getDAGName());
    assertEquals(vertexName, procContext.getTaskVertexName());
    assertEquals(vertexId.getId(), procContext.getTaskVertexIndex());
    assertTrue(Arrays.equals(localDirs, procContext.getWorkDirs()));
    // test auto call of notifyProgress
    procContext.setProgress(0.1f);
    verify(mockTask, times(1)).notifyProgressInvocation();
    sharedExecutor.shutdown();
}