Examples with InputSpec org.apache.tez.runtime.api.impl.InputSpec used on opensource projects

Example 11 with InputSpec

use of org.apache.tez.runtime.api.impl.InputSpec in project tez by apache.

the class ProtoConverters method convertTaskSpecToProto.

public static TaskSpecProto convertTaskSpecToProto(TaskSpec taskSpec) {
    Builder builder = TaskSpecProto.newBuilder();
    builder.setTaskAttemptIdString(taskSpec.getTaskAttemptID().toString());
    builder.setDagName(taskSpec.getDAGName());
    builder.setVertexName(taskSpec.getVertexName());
    builder.setVertexParallelism(taskSpec.getVertexParallelism());
    if (taskSpec.getProcessorDescriptor() != null) {
        builder.setProcessorDescriptor(DagTypeConverters.convertToDAGPlan(taskSpec.getProcessorDescriptor()));
    }
    if (taskSpec.getInputs() != null && !taskSpec.getInputs().isEmpty()) {
        for (InputSpec inputSpec : taskSpec.getInputs()) {
            builder.addInputSpecs(convertInputSpecToProto(inputSpec));
        }
    }
    if (taskSpec.getOutputs() != null && !taskSpec.getOutputs().isEmpty()) {
        for (OutputSpec outputSpec : taskSpec.getOutputs()) {
            builder.addOutputSpecs(convertOutputSpecToProto(outputSpec));
        }
    }
    if (taskSpec.getGroupInputs() != null && !taskSpec.getGroupInputs().isEmpty()) {
        for (GroupInputSpec groupInputSpec : taskSpec.getGroupInputs()) {
            builder.addGroupedInputSpecs(convertGroupInputSpecToProto(groupInputSpec));
        }
    }
    if (taskSpec.getTaskConf() != null) {
        ConfigurationProto.Builder confBuilder = ConfigurationProto.newBuilder();
        Iterator<Entry<String, String>> iter = taskSpec.getTaskConf().iterator();
        while (iter.hasNext()) {
            Entry<String, String> entry = iter.next();
            confBuilder.addConfKeyValues(PlanKeyValuePair.newBuilder().setKey(entry.getKey()).setValue(entry.getValue()).build());
        }
        builder.setTaskConf(confBuilder.build());
    }
    return builder.build();
}

Example 12 with InputSpec

use of org.apache.tez.runtime.api.impl.InputSpec in project hive by apache.

the class Converters method getTaskSpecfromProto.

public static TaskSpec getTaskSpecfromProto(SignableVertexSpec vectorProto, int fragmentNum, int attemptNum, TezTaskAttemptID attemptId) {
    TezTaskAttemptID taskAttemptID = attemptId != null ? attemptId : createTaskAttemptId(vectorProto.getQueryIdentifier(), vectorProto.getVertexIndex(), fragmentNum, attemptNum);
    ProcessorDescriptor processorDescriptor = null;
    if (vectorProto.hasProcessorDescriptor()) {
        processorDescriptor = convertProcessorDescriptorFromProto(vectorProto.getProcessorDescriptor());
    }
    List<InputSpec> inputSpecList = new ArrayList<InputSpec>(vectorProto.getInputSpecsCount());
    if (vectorProto.getInputSpecsCount() > 0) {
        for (IOSpecProto inputSpecProto : vectorProto.getInputSpecsList()) {
            inputSpecList.add(getInputSpecFromProto(inputSpecProto));
        }
    }
    List<OutputSpec> outputSpecList = new ArrayList<OutputSpec>(vectorProto.getOutputSpecsCount());
    if (vectorProto.getOutputSpecsCount() > 0) {
        for (IOSpecProto outputSpecProto : vectorProto.getOutputSpecsList()) {
            outputSpecList.add(getOutputSpecFromProto(outputSpecProto));
        }
    }
    List<GroupInputSpec> groupInputSpecs = new ArrayList<GroupInputSpec>(vectorProto.getGroupedInputSpecsCount());
    if (vectorProto.getGroupedInputSpecsCount() > 0) {
        for (GroupInputSpecProto groupInputSpecProto : vectorProto.getGroupedInputSpecsList()) {
            groupInputSpecs.add(getGroupInputSpecFromProto(groupInputSpecProto));
        }
    }
    TaskSpec taskSpec = new TaskSpec(taskAttemptID, vectorProto.getDagName(), vectorProto.getVertexName(), vectorProto.getVertexParallelism(), processorDescriptor, inputSpecList, outputSpecList, groupInputSpecs);
    return taskSpec;
}

Example 13 with InputSpec

use of org.apache.tez.runtime.api.impl.InputSpec in project hive by apache.

the class Converters method constructSignableVertexSpec.

public static SignableVertexSpec.Builder constructSignableVertexSpec(TaskSpec taskSpec, QueryIdentifierProto queryIdentifierProto, String tokenIdentifier, String user, String hiveQueryIdString) {
    TezTaskAttemptID tId = taskSpec.getTaskAttemptID();
    SignableVertexSpec.Builder builder = SignableVertexSpec.newBuilder();
    builder.setQueryIdentifier(queryIdentifierProto);
    builder.setHiveQueryId(hiveQueryIdString);
    builder.setVertexIndex(tId.getTaskID().getVertexID().getId());
    builder.setDagName(taskSpec.getDAGName());
    builder.setVertexName(taskSpec.getVertexName());
    builder.setVertexParallelism(taskSpec.getVertexParallelism());
    builder.setTokenIdentifier(tokenIdentifier);
    builder.setUser(user);
    if (taskSpec.getProcessorDescriptor() != null) {
        builder.setProcessorDescriptor(convertToProto(taskSpec.getProcessorDescriptor()));
    }
    if (taskSpec.getInputs() != null && !taskSpec.getInputs().isEmpty()) {
        for (InputSpec inputSpec : taskSpec.getInputs()) {
            builder.addInputSpecs(convertInputSpecToProto(inputSpec));
        }
    }
    if (taskSpec.getOutputs() != null && !taskSpec.getOutputs().isEmpty()) {
        for (OutputSpec outputSpec : taskSpec.getOutputs()) {
            builder.addOutputSpecs(convertOutputSpecToProto(outputSpec));
        }
    }
    if (taskSpec.getGroupInputs() != null && !taskSpec.getGroupInputs().isEmpty()) {
        for (GroupInputSpec groupInputSpec : taskSpec.getGroupInputs()) {
            builder.addGroupedInputSpecs(convertGroupInputSpecToProto(groupInputSpec));
        }
    }
    return builder;
}

Example 14 with InputSpec

use of org.apache.tez.runtime.api.impl.InputSpec in project hive by apache.

the class Converters method getInputSpecFromProto.

private static InputSpec getInputSpecFromProto(IOSpecProto inputSpecProto) {
    InputDescriptor inputDescriptor = null;
    if (inputSpecProto.hasIoDescriptor()) {
        inputDescriptor = convertInputDescriptorFromProto(inputSpecProto.getIoDescriptor());
    }
    InputSpec inputSpec = new InputSpec(inputSpecProto.getConnectedVertexName(), inputDescriptor, inputSpecProto.getPhysicalEdgeCount());
    return inputSpec;
}

Example 15 with InputSpec

use of org.apache.tez.runtime.api.impl.InputSpec in project hive by apache.

the class TestConverters method testTaskSpecToFragmentSpec.

@Test(timeout = 10000)
public void testTaskSpecToFragmentSpec() {
    ByteBuffer procBb = ByteBuffer.allocate(4);
    procBb.putInt(0, 200);
    UserPayload processorPayload = UserPayload.create(procBb);
    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create("fakeProcessorName").setUserPayload(processorPayload);
    ByteBuffer input1Bb = ByteBuffer.allocate(4);
    input1Bb.putInt(0, 300);
    UserPayload input1Payload = UserPayload.create(input1Bb);
    InputDescriptor id1 = InputDescriptor.create("input1ClassName").setUserPayload(input1Payload);
    InputSpec inputSpec1 = new InputSpec("sourceVertexName1", id1, 33);
    InputSpec inputSpec2 = new InputSpec("sourceVertexName2", id1, 44);
    List<InputSpec> inputSpecList = Lists.newArrayList(inputSpec1, inputSpec2);
    ByteBuffer output1Bb = ByteBuffer.allocate(4);
    output1Bb.putInt(0, 400);
    UserPayload output1Payload = UserPayload.create(output1Bb);
    OutputDescriptor od1 = OutputDescriptor.create("output1ClassName").setUserPayload(output1Payload);
    OutputSpec outputSpec1 = new OutputSpec("destVertexName1", od1, 55);
    OutputSpec outputSpec2 = new OutputSpec("destVertexName2", od1, 66);
    List<OutputSpec> outputSpecList = Lists.newArrayList(outputSpec1, outputSpec2);
    ApplicationId appId = ApplicationId.newInstance(1000, 100);
    TezDAGID tezDagId = TezDAGID.getInstance(appId, 300);
    TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400);
    TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500);
    TezTaskAttemptID tezTaskAttemptId = TezTaskAttemptID.getInstance(tezTaskId, 600);
    TaskSpec taskSpec = new TaskSpec(tezTaskAttemptId, "dagName", "vertexName", 10, processorDescriptor, inputSpecList, outputSpecList, null);
    QueryIdentifierProto queryIdentifierProto = QueryIdentifierProto.newBuilder().setApplicationIdString(appId.toString()).setAppAttemptNumber(333).setDagIndex(300).build();
    SignableVertexSpec vertexProto = Converters.constructSignableVertexSpec(taskSpec, queryIdentifierProto, "", "", "hiveQueryId").build();
    assertEquals("dagName", vertexProto.getDagName());
    assertEquals("vertexName", vertexProto.getVertexName());
    assertEquals("hiveQueryId", vertexProto.getHiveQueryId());
    assertEquals(appId.toString(), vertexProto.getQueryIdentifier().getApplicationIdString());
    assertEquals(tezDagId.getId(), vertexProto.getQueryIdentifier().getDagIndex());
    assertEquals(333, vertexProto.getQueryIdentifier().getAppAttemptNumber());
    assertEquals(tezVertexId.getId(), vertexProto.getVertexIndex());
    assertEquals(processorDescriptor.getClassName(), vertexProto.getProcessorDescriptor().getClassName());
    assertEquals(processorDescriptor.getUserPayload().getPayload(), vertexProto.getProcessorDescriptor().getUserPayload().getUserPayload().asReadOnlyByteBuffer());
    assertEquals(2, vertexProto.getInputSpecsCount());
    assertEquals(2, vertexProto.getOutputSpecsCount());
    verifyInputSpecAndProto(inputSpec1, vertexProto.getInputSpecs(0));
    verifyInputSpecAndProto(inputSpec2, vertexProto.getInputSpecs(1));
    verifyOutputSpecAndProto(outputSpec1, vertexProto.getOutputSpecs(0));
    verifyOutputSpecAndProto(outputSpec2, vertexProto.getOutputSpecs(1));
}