Examples with OutputDescriptor org.apache.tez.dag.api.OutputDescriptor used on opensource projects

Example 16 with OutputDescriptor

use of org.apache.tez.dag.api.OutputDescriptor in project hive by apache.

the class Converters method getOutputSpecFromProto.

private static OutputSpec getOutputSpecFromProto(IOSpecProto outputSpecProto) {
    OutputDescriptor outputDescriptor = null;
    if (outputSpecProto.hasIoDescriptor()) {
        outputDescriptor = convertOutputDescriptorFromProto(outputSpecProto.getIoDescriptor());
    }
    OutputSpec outputSpec = new OutputSpec(outputSpecProto.getConnectedVertexName(), outputDescriptor, outputSpecProto.getPhysicalEdgeCount());
    return outputSpec;
}

Example 17 with OutputDescriptor

use of org.apache.tez.dag.api.OutputDescriptor in project hive by apache.

the class TestConverters method testTaskSpecToFragmentSpec.

@Test(timeout = 10000)
public void testTaskSpecToFragmentSpec() {
    ByteBuffer procBb = ByteBuffer.allocate(4);
    procBb.putInt(0, 200);
    UserPayload processorPayload = UserPayload.create(procBb);
    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create("fakeProcessorName").setUserPayload(processorPayload);
    ByteBuffer input1Bb = ByteBuffer.allocate(4);
    input1Bb.putInt(0, 300);
    UserPayload input1Payload = UserPayload.create(input1Bb);
    InputDescriptor id1 = InputDescriptor.create("input1ClassName").setUserPayload(input1Payload);
    InputSpec inputSpec1 = new InputSpec("sourceVertexName1", id1, 33);
    InputSpec inputSpec2 = new InputSpec("sourceVertexName2", id1, 44);
    List<InputSpec> inputSpecList = Lists.newArrayList(inputSpec1, inputSpec2);
    ByteBuffer output1Bb = ByteBuffer.allocate(4);
    output1Bb.putInt(0, 400);
    UserPayload output1Payload = UserPayload.create(output1Bb);
    OutputDescriptor od1 = OutputDescriptor.create("output1ClassName").setUserPayload(output1Payload);
    OutputSpec outputSpec1 = new OutputSpec("destVertexName1", od1, 55);
    OutputSpec outputSpec2 = new OutputSpec("destVertexName2", od1, 66);
    List<OutputSpec> outputSpecList = Lists.newArrayList(outputSpec1, outputSpec2);
    ApplicationId appId = ApplicationId.newInstance(1000, 100);
    TezDAGID tezDagId = TezDAGID.getInstance(appId, 300);
    TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400);
    TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500);
    TezTaskAttemptID tezTaskAttemptId = TezTaskAttemptID.getInstance(tezTaskId, 600);
    TaskSpec taskSpec = new TaskSpec(tezTaskAttemptId, "dagName", "vertexName", 10, processorDescriptor, inputSpecList, outputSpecList, null);
    QueryIdentifierProto queryIdentifierProto = QueryIdentifierProto.newBuilder().setApplicationIdString(appId.toString()).setAppAttemptNumber(333).setDagIndex(300).build();
    SignableVertexSpec vertexProto = Converters.constructSignableVertexSpec(taskSpec, queryIdentifierProto, "", "", "hiveQueryId").build();
    assertEquals("dagName", vertexProto.getDagName());
    assertEquals("vertexName", vertexProto.getVertexName());
    assertEquals("hiveQueryId", vertexProto.getHiveQueryId());
    assertEquals(appId.toString(), vertexProto.getQueryIdentifier().getApplicationIdString());
    assertEquals(tezDagId.getId(), vertexProto.getQueryIdentifier().getDagIndex());
    assertEquals(333, vertexProto.getQueryIdentifier().getAppAttemptNumber());
    assertEquals(tezVertexId.getId(), vertexProto.getVertexIndex());
    assertEquals(processorDescriptor.getClassName(), vertexProto.getProcessorDescriptor().getClassName());
    assertEquals(processorDescriptor.getUserPayload().getPayload(), vertexProto.getProcessorDescriptor().getUserPayload().getUserPayload().asReadOnlyByteBuffer());
    assertEquals(2, vertexProto.getInputSpecsCount());
    assertEquals(2, vertexProto.getOutputSpecsCount());
    verifyInputSpecAndProto(inputSpec1, vertexProto.getInputSpecs(0));
    verifyInputSpecAndProto(inputSpec2, vertexProto.getInputSpecs(1));
    verifyOutputSpecAndProto(outputSpec1, vertexProto.getOutputSpecs(0));
    verifyOutputSpecAndProto(outputSpec2, vertexProto.getOutputSpecs(1));
}

Example 18 with OutputDescriptor

use of org.apache.tez.dag.api.OutputDescriptor in project tez by apache.

the class TestCommit method createDAGPlan_SingleVertexWith2Committer.

// used for route event error in VM
private DAGPlan createDAGPlan_SingleVertexWith2Committer(boolean commit1Succeed, boolean commit2Succeed, boolean customVM) throws IOException {
    LOG.info("Setting up group dag plan");
    int dummyTaskCount = 1;
    Resource dummyTaskResource = Resource.newInstance(1, 1);
    org.apache.tez.dag.api.Vertex v1 = org.apache.tez.dag.api.Vertex.create("vertex1", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource);
    if (customVM) {
        v1.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(FailOnVMEventReceivedlVertexManager.class.getName()));
    }
    OutputCommitterDescriptor ocd1 = OutputCommitterDescriptor.create(CountingOutputCommitter.class.getName()).setUserPayload(UserPayload.create(ByteBuffer.wrap(new CountingOutputCommitter.CountingOutputCommitterConfig(!commit1Succeed, true).toUserPayload())));
    OutputCommitterDescriptor ocd2 = OutputCommitterDescriptor.create(CountingOutputCommitter.class.getName()).setUserPayload(UserPayload.create(ByteBuffer.wrap(new CountingOutputCommitter.CountingOutputCommitterConfig(!commit2Succeed, true).toUserPayload())));
    DAG dag = DAG.create("testDag");
    dag.addVertex(v1);
    OutputDescriptor outDesc = OutputDescriptor.create("output.class");
    v1.addDataSink("v1Out_1", DataSinkDescriptor.create(outDesc, ocd1, null));
    v1.addDataSink("v1Out_2", DataSinkDescriptor.create(outDesc, ocd2, null));
    return dag.createDag(conf, null, null, null, true);
}

Example 19 with OutputDescriptor

use of org.apache.tez.dag.api.OutputDescriptor in project tez by apache.

the class TestDAGImpl method createGroupDAGPlan.

// Create a plan with 3 vertices: A, B, C. Group(A,B)->C
static DAGPlan createGroupDAGPlan() {
    LOG.info("Setting up group dag plan");
    int dummyTaskCount = 1;
    Resource dummyTaskResource = Resource.newInstance(1, 1);
    org.apache.tez.dag.api.Vertex v1 = org.apache.tez.dag.api.Vertex.create("vertex1", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource);
    org.apache.tez.dag.api.Vertex v2 = org.apache.tez.dag.api.Vertex.create("vertex2", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource);
    org.apache.tez.dag.api.Vertex v3 = org.apache.tez.dag.api.Vertex.create("vertex3", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource);
    DAG dag = DAG.create("testDag");
    String groupName1 = "uv12";
    OutputCommitterDescriptor ocd = OutputCommitterDescriptor.create(TotalCountingOutputCommitter.class.getName());
    org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
    OutputDescriptor outDesc = OutputDescriptor.create("output.class");
    uv12.addDataSink("uvOut", DataSinkDescriptor.create(outDesc, ocd, null));
    v3.addDataSink("uvOut", DataSinkDescriptor.create(outDesc, ocd, null));
    GroupInputEdge e1 = GroupInputEdge.create(uv12, v3, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("dummy output class"), InputDescriptor.create("dummy input class")), InputDescriptor.create("merge.class"));
    dag.addVertex(v1);
    dag.addVertex(v2);
    dag.addVertex(v3);
    dag.addEdge(e1);
    return dag.createDag(conf, null, null, null, true);
}

Example 20 with OutputDescriptor

use of org.apache.tez.dag.api.OutputDescriptor in project tez by apache.

the class LogicalIOProcessorRuntimeTask method createOutput.

private LogicalOutput createOutput(OutputSpec outputSpec, OutputContext outputContext) throws TezException {
    OutputDescriptor outputDesc = outputSpec.getOutputDescriptor();
    Output output = ReflectionUtils.createClazzInstance(outputDesc.getClassName(), new Class[] { OutputContext.class, Integer.TYPE }, new Object[] { outputContext, outputSpec.getPhysicalEdgeCount() });
    if (!(output instanceof LogicalOutput)) {
        throw new TezUncheckedException(output.getClass().getName() + " is not a sub-type of LogicalOutput." + " Only LogicalOutput sub-types supported by LogicalIOProcessor.");
    }
    return (LogicalOutput) output;
}