Examples with TezDAGID org.apache.tez.dag.records.TezDAGID used on opensource projects

Example 61 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project hive by apache.

the class TestConverters method testFragmentSpecToTaskSpec.

@Test(timeout = 10000)
public void testFragmentSpecToTaskSpec() {
    ByteBuffer procBb = ByteBuffer.allocate(4);
    procBb.putInt(0, 200);
    ByteBuffer input1Bb = ByteBuffer.allocate(4);
    input1Bb.putInt(0, 300);
    ByteBuffer output1Bb = ByteBuffer.allocate(4);
    output1Bb.putInt(0, 400);
    ApplicationId appId = ApplicationId.newInstance(1000, 100);
    TezDAGID tezDagId = TezDAGID.getInstance(appId, 300);
    TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400);
    TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500);
    TezTaskAttemptID tezTaskAttemptId = TezTaskAttemptID.getInstance(tezTaskId, 600);
    QueryIdentifierProto queryIdentifierProto = QueryIdentifierProto.newBuilder().setApplicationIdString(appId.toString()).setAppAttemptNumber(333).setDagIndex(tezDagId.getId()).build();
    SignableVertexSpec.Builder builder = SignableVertexSpec.newBuilder();
    builder.setQueryIdentifier(queryIdentifierProto);
    builder.setHiveQueryId("hiveQueryId");
    builder.setVertexIndex(tezVertexId.getId());
    builder.setDagName("dagName");
    builder.setVertexName("vertexName");
    builder.setProcessorDescriptor(EntityDescriptorProto.newBuilder().setClassName("fakeProcessorName").setUserPayload(UserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(procBb))));
    builder.addInputSpecs(IOSpecProto.newBuilder().setConnectedVertexName("sourceVertexName1").setPhysicalEdgeCount(33).setIoDescriptor(EntityDescriptorProto.newBuilder().setClassName("input1ClassName").setUserPayload(UserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(input1Bb)))));
    builder.addInputSpecs(IOSpecProto.newBuilder().setConnectedVertexName("sourceVertexName2").setPhysicalEdgeCount(44).setIoDescriptor(EntityDescriptorProto.newBuilder().setClassName("input1ClassName").setUserPayload(UserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(input1Bb)))));
    builder.addOutputSpecs(IOSpecProto.newBuilder().setConnectedVertexName("destVertexName1").setPhysicalEdgeCount(55).setIoDescriptor(EntityDescriptorProto.newBuilder().setClassName("outputClassName").setUserPayload(UserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(output1Bb)))));
    builder.addOutputSpecs(IOSpecProto.newBuilder().setConnectedVertexName("destVertexName2").setPhysicalEdgeCount(66).setIoDescriptor(EntityDescriptorProto.newBuilder().setClassName("outputClassName").setUserPayload(UserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(output1Bb)))));
    SignableVertexSpec vertexProto = builder.build();
    TaskSpec taskSpec = Converters.getTaskSpecfromProto(vertexProto, 0, 0, tezTaskAttemptId);
    assertEquals("dagName", taskSpec.getDAGName());
    assertEquals("vertexName", taskSpec.getVertexName());
    assertEquals(tezTaskAttemptId, taskSpec.getTaskAttemptID());
    assertEquals("fakeProcessorName", taskSpec.getProcessorDescriptor().getClassName());
    byte[] serialized = new byte[taskSpec.getProcessorDescriptor().getUserPayload().getPayload().remaining()];
    taskSpec.getProcessorDescriptor().getUserPayload().getPayload().get(serialized);
    assertArrayEquals(procBb.array(), serialized);
    assertEquals(2, taskSpec.getInputs().size());
    assertEquals(2, taskSpec.getOutputs().size());
    verifyInputSpecAndProto(taskSpec.getInputs().get(0), vertexProto.getInputSpecs(0));
    verifyInputSpecAndProto(taskSpec.getInputs().get(1), vertexProto.getInputSpecs(1));
    verifyOutputSpecAndProto(taskSpec.getOutputs().get(0), vertexProto.getOutputSpecs(0));
    verifyOutputSpecAndProto(taskSpec.getOutputs().get(1), vertexProto.getOutputSpecs(1));
}

Example 62 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project hive by apache.

the class TestConverters method testTaskSpecToFragmentSpec.

@Test(timeout = 10000)
public void testTaskSpecToFragmentSpec() {
    ByteBuffer procBb = ByteBuffer.allocate(4);
    procBb.putInt(0, 200);
    UserPayload processorPayload = UserPayload.create(procBb);
    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create("fakeProcessorName").setUserPayload(processorPayload);
    ByteBuffer input1Bb = ByteBuffer.allocate(4);
    input1Bb.putInt(0, 300);
    UserPayload input1Payload = UserPayload.create(input1Bb);
    InputDescriptor id1 = InputDescriptor.create("input1ClassName").setUserPayload(input1Payload);
    InputSpec inputSpec1 = new InputSpec("sourceVertexName1", id1, 33);
    InputSpec inputSpec2 = new InputSpec("sourceVertexName2", id1, 44);
    List<InputSpec> inputSpecList = Lists.newArrayList(inputSpec1, inputSpec2);
    ByteBuffer output1Bb = ByteBuffer.allocate(4);
    output1Bb.putInt(0, 400);
    UserPayload output1Payload = UserPayload.create(output1Bb);
    OutputDescriptor od1 = OutputDescriptor.create("output1ClassName").setUserPayload(output1Payload);
    OutputSpec outputSpec1 = new OutputSpec("destVertexName1", od1, 55);
    OutputSpec outputSpec2 = new OutputSpec("destVertexName2", od1, 66);
    List<OutputSpec> outputSpecList = Lists.newArrayList(outputSpec1, outputSpec2);
    ApplicationId appId = ApplicationId.newInstance(1000, 100);
    TezDAGID tezDagId = TezDAGID.getInstance(appId, 300);
    TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400);
    TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500);
    TezTaskAttemptID tezTaskAttemptId = TezTaskAttemptID.getInstance(tezTaskId, 600);
    TaskSpec taskSpec = new TaskSpec(tezTaskAttemptId, "dagName", "vertexName", 10, processorDescriptor, inputSpecList, outputSpecList, null);
    QueryIdentifierProto queryIdentifierProto = QueryIdentifierProto.newBuilder().setApplicationIdString(appId.toString()).setAppAttemptNumber(333).setDagIndex(300).build();
    SignableVertexSpec vertexProto = Converters.constructSignableVertexSpec(taskSpec, queryIdentifierProto, "", "", "hiveQueryId").build();
    assertEquals("dagName", vertexProto.getDagName());
    assertEquals("vertexName", vertexProto.getVertexName());
    assertEquals("hiveQueryId", vertexProto.getHiveQueryId());
    assertEquals(appId.toString(), vertexProto.getQueryIdentifier().getApplicationIdString());
    assertEquals(tezDagId.getId(), vertexProto.getQueryIdentifier().getDagIndex());
    assertEquals(333, vertexProto.getQueryIdentifier().getAppAttemptNumber());
    assertEquals(tezVertexId.getId(), vertexProto.getVertexIndex());
    assertEquals(processorDescriptor.getClassName(), vertexProto.getProcessorDescriptor().getClassName());
    assertEquals(processorDescriptor.getUserPayload().getPayload(), vertexProto.getProcessorDescriptor().getUserPayload().getUserPayload().asReadOnlyByteBuffer());
    assertEquals(2, vertexProto.getInputSpecsCount());
    assertEquals(2, vertexProto.getOutputSpecsCount());
    verifyInputSpecAndProto(inputSpec1, vertexProto.getInputSpecs(0));
    verifyInputSpecAndProto(inputSpec2, vertexProto.getInputSpecs(1));
    verifyOutputSpecAndProto(outputSpec1, vertexProto.getOutputSpecs(0));
    verifyOutputSpecAndProto(outputSpec2, vertexProto.getOutputSpecs(1));
}

Example 63 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project hive by apache.

the class TaskExecutorTestHelpers method createSubmitWorkRequestProto.

public static SubmitWorkRequestProto createSubmitWorkRequestProto(int fragmentNumber, int selfAndUpstreamParallelism, int selfAndUpstreamComplete, long firstAttemptStartTime, long currentAttemptStartTime, int withinDagPriority, String dagName, int dagId, boolean isGuaranteed) {
    ApplicationId appId = ApplicationId.newInstance(9999, 72);
    TezDAGID dag = TezDAGID.getInstance(appId, 1);
    TezVertexID vId = TezVertexID.getInstance(dag, dagId);
    return SubmitWorkRequestProto.newBuilder().setAttemptNumber(0).setFragmentNumber(fragmentNumber).setWorkSpec(VertexOrBinary.newBuilder().setVertex(SignableVertexSpec.newBuilder().setDagName(dagName).setHiveQueryId(dagName).setUser("MockUser").setTokenIdentifier("MockToken_1").setQueryIdentifier(QueryIdentifierProto.newBuilder().setApplicationIdString(appId.toString()).setAppAttemptNumber(0).setDagIndex(dag.getId()).build()).setVertexIndex(vId.getId()).setVertexName("MockVertex").setProcessorDescriptor(LlapDaemonProtocolProtos.EntityDescriptorProto.newBuilder().setClassName("MockProcessor").build()).build()).build()).setAmHost("localhost").setAmPort(12345).setContainerIdString("MockContainer_1").setIsGuaranteed(isGuaranteed).setFragmentRuntimeInfo(LlapDaemonProtocolProtos.FragmentRuntimeInfo.newBuilder().setFirstAttemptStartTime(firstAttemptStartTime).setCurrentAttemptStartTime(currentAttemptStartTime).setNumSelfAndUpstreamTasks(selfAndUpstreamParallelism).setNumSelfAndUpstreamCompletedTasks(selfAndUpstreamComplete).setWithinDagPriority(withinDagPriority).build()).build();
}

Example 64 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project hive by apache.

the class TaskSpecBuilder method constructTaskSpec.

public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) {
    Vertex vertex = dag.getVertex(vertexName);
    ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor();
    List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs();
    List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs();
    Preconditions.checkState(inputs.size() == 1);
    Preconditions.checkState(outputs.size() == 1);
    List<InputSpec> inputSpecs = new ArrayList<>();
    for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) {
        InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1);
        inputSpecs.add(inputSpec);
    }
    List<OutputSpec> outputSpecs = new ArrayList<>();
    for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) {
        OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1);
        outputSpecs.add(outputSpec);
    }
    TezDAGID dagId = TezDAGID.getInstance(appId, 0);
    TezVertexID vertexId = TezVertexID.getInstance(dagId, 0);
    TezTaskID taskId = TezTaskID.getInstance(vertexId, index);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0);
    return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null);
}

Example 65 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project hive by apache.

the class TaskExecutorTestHelpers method createSubmitWorkRequestProto.

public static SubmitWorkRequestProto createSubmitWorkRequestProto(int fragmentNumber, int selfAndUpstreamParallelism, int selfAndUpstreamComplete, long firstAttemptStartTime, long currentAttemptStartTime, int withinDagPriority, String dagName) {
    ApplicationId appId = ApplicationId.newInstance(9999, 72);
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vId = TezVertexID.getInstance(dagId, 35);
    return SubmitWorkRequestProto.newBuilder().setAttemptNumber(0).setFragmentNumber(fragmentNumber).setWorkSpec(VertexOrBinary.newBuilder().setVertex(SignableVertexSpec.newBuilder().setDagName(dagName).setHiveQueryId(dagName).setUser("MockUser").setTokenIdentifier("MockToken_1").setQueryIdentifier(QueryIdentifierProto.newBuilder().setApplicationIdString(appId.toString()).setAppAttemptNumber(0).setDagIndex(dagId.getId()).build()).setVertexIndex(vId.getId()).setVertexName("MockVertex").setProcessorDescriptor(LlapDaemonProtocolProtos.EntityDescriptorProto.newBuilder().setClassName("MockProcessor").build()).build()).build()).setAmHost("localhost").setAmPort(12345).setContainerIdString("MockContainer_1").setFragmentRuntimeInfo(LlapDaemonProtocolProtos.FragmentRuntimeInfo.newBuilder().setFirstAttemptStartTime(firstAttemptStartTime).setCurrentAttemptStartTime(currentAttemptStartTime).setNumSelfAndUpstreamTasks(selfAndUpstreamParallelism).setNumSelfAndUpstreamCompletedTasks(selfAndUpstreamComplete).setWithinDagPriority(withinDagPriority).build()).build();
}