Examples with InputDescriptor org.apache.tez.dag.api.InputDescriptor used on opensource projects

Example 21 with InputDescriptor

use of org.apache.tez.dag.api.InputDescriptor in project tez by apache.

the class TestRootInputVertexManager method testConfigureFromMultipleInputs.

@Test(timeout = 5000)
public void testConfigureFromMultipleInputs() throws IOException {
    VertexManagerPluginContext context = mock(VertexManagerPluginContext.class);
    TezConfiguration conf = new TezConfiguration();
    UserPayload vertexPayload = TezUtils.createUserPayloadFromConf(conf);
    doReturn("vertex1").when(context).getVertexName();
    doReturn(-1).when(context).getVertexNumTasks(eq("vertex1"));
    doReturn(vertexPayload).when(context).getUserPayload();
    RootInputVertexManager rootInputVertexManager = new RootInputVertexManager(context);
    rootInputVertexManager.initialize();
    InputDescriptor id1 = mock(InputDescriptor.class);
    List<Event> events1 = new LinkedList<Event>();
    InputConfigureVertexTasksEvent diEvent11 = InputConfigureVertexTasksEvent.create(1, null, null);
    events1.add(diEvent11);
    rootInputVertexManager.onRootVertexInitialized("input1", id1, events1);
    // All good so far, single input only.
    InputDescriptor id2 = mock(InputDescriptor.class);
    List<Event> events2 = new LinkedList<Event>();
    InputConfigureVertexTasksEvent diEvent21 = InputConfigureVertexTasksEvent.create(1, null, null);
    events2.add(diEvent21);
    try {
        // Should fail due to second input
        rootInputVertexManager.onRootVertexInitialized("input2", id2, events2);
        fail("Expecting failure in case of multiple inputs attempting to send events");
    } catch (IllegalStateException e) {
        assertTrue(e.getMessage().startsWith("RootInputVertexManager cannot configure multiple inputs. Use a custom VertexManager"));
    }
}

Example 22 with InputDescriptor

use of org.apache.tez.dag.api.InputDescriptor in project tez by apache.

the class YARNRunner method configureMRInputWithLegacySplitsGenerated.

@Private
private static DataSourceDescriptor configureMRInputWithLegacySplitsGenerated(Configuration conf, boolean useLegacyInput) {
    InputDescriptor inputDescriptor;
    try {
        inputDescriptor = InputDescriptor.create(useLegacyInput ? MRInputLegacy.class.getName() : MRInput.class.getName()).setUserPayload(MRInputHelpersInternal.createMRInputPayload(conf, null));
    } catch (IOException e) {
        throw new TezUncheckedException(e);
    }
    DataSourceDescriptor dsd = DataSourceDescriptor.create(inputDescriptor, null, null);
    if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
        dsd.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf));
    }
    return dsd;
}

Example 23 with InputDescriptor

use of org.apache.tez.dag.api.InputDescriptor in project tez by apache.

the class ProtoConverters method getInputSpecFromProto.

public static InputSpec getInputSpecFromProto(IOSpecProto inputSpecProto) {
    InputDescriptor inputDescriptor = null;
    if (inputSpecProto.hasIoDescriptor()) {
        inputDescriptor = DagTypeConverters.convertInputDescriptorFromDAGPlan(inputSpecProto.getIoDescriptor());
    }
    InputSpec inputSpec = new InputSpec(inputSpecProto.getConnectedVertexName(), inputDescriptor, inputSpecProto.getPhysicalEdgeCount());
    return inputSpec;
}

Example 24 with InputDescriptor

use of org.apache.tez.dag.api.InputDescriptor in project tez by apache.

the class VertexInitializedEvent method fromProto.

public void fromProto(RecoveryProtos.VertexInitializedProto proto) throws IOException {
    this.vertexID = TezVertexID.fromString(proto.getVertexId());
    this.vertexName = proto.getVertexName();
    this.initRequestedTime = proto.getInitRequestedTime();
    this.initedTime = proto.getInitTime();
    this.numTasks = proto.getNumTasks();
    if (proto.getInputsCount() > 0) {
        this.additionalInputs = new LinkedHashMap<String, RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>>();
        for (RootInputLeafOutputProto inputProto : proto.getInputsList()) {
            RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input = new RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>(inputProto.getName(), DagTypeConverters.convertInputDescriptorFromDAGPlan(inputProto.getIODescriptor()), inputProto.hasControllerDescriptor() ? DagTypeConverters.convertInputInitializerDescriptorFromDAGPlan(inputProto.getControllerDescriptor()) : null);
            additionalInputs.put(input.getName(), input);
        }
    }
    int eventCount = proto.getInitGeneratedEventsCount();
    if (eventCount > 0) {
        this.initGeneratedEvents = Lists.newArrayListWithCapacity(eventCount);
    }
    for (TezEventProto eventProto : proto.getInitGeneratedEventsList()) {
        this.initGeneratedEvents.add(TezEventUtils.fromProto(eventProto));
    }
}

Example 25 with InputDescriptor

use of org.apache.tez.dag.api.InputDescriptor in project tez by apache.

the class LogicalIOProcessorRuntimeTask method createInput.

private LogicalInput createInput(InputSpec inputSpec, InputContext inputContext) throws TezException {
    InputDescriptor inputDesc = inputSpec.getInputDescriptor();
    Input input = ReflectionUtils.createClazzInstance(inputDesc.getClassName(), new Class[] { InputContext.class, Integer.TYPE }, new Object[] { inputContext, inputSpec.getPhysicalEdgeCount() });
    if (!(input instanceof LogicalInput)) {
        throw new TezUncheckedException(inputDesc.getClass().getName() + " is not a sub-type of LogicalInput." + " Only LogicalInput sub-types supported by LogicalIOProcessor.");
    }
    return (LogicalInput) input;
}