Examples with InputSpec org.apache.tez.runtime.api.impl.InputSpec used on opensource projects

Example 21 with InputSpec

use of org.apache.tez.runtime.api.impl.InputSpec in project tez by apache.

the class ProtoConverters method getInputSpecFromProto.

public static InputSpec getInputSpecFromProto(IOSpecProto inputSpecProto) {
    InputDescriptor inputDescriptor = null;
    if (inputSpecProto.hasIoDescriptor()) {
        inputDescriptor = DagTypeConverters.convertInputDescriptorFromDAGPlan(inputSpecProto.getIoDescriptor());
    }
    InputSpec inputSpec = new InputSpec(inputSpecProto.getConnectedVertexName(), inputDescriptor, inputSpecProto.getPhysicalEdgeCount());
    return inputSpec;
}

Example 22 with InputSpec

use of org.apache.tez.runtime.api.impl.InputSpec in project tez by apache.

the class ProtoConverters method getTaskSpecfromProto.

public static TaskSpec getTaskSpecfromProto(TaskSpecProto taskSpecProto) {
    TezTaskAttemptID taskAttemptID = TezTaskAttemptID.fromString(taskSpecProto.getTaskAttemptIdString());
    ProcessorDescriptor processorDescriptor = null;
    if (taskSpecProto.hasProcessorDescriptor()) {
        processorDescriptor = DagTypeConverters.convertProcessorDescriptorFromDAGPlan(taskSpecProto.getProcessorDescriptor());
    }
    List<InputSpec> inputSpecList = new ArrayList<InputSpec>(taskSpecProto.getInputSpecsCount());
    if (taskSpecProto.getInputSpecsCount() > 0) {
        for (IOSpecProto inputSpecProto : taskSpecProto.getInputSpecsList()) {
            inputSpecList.add(getInputSpecFromProto(inputSpecProto));
        }
    }
    List<OutputSpec> outputSpecList = new ArrayList<OutputSpec>(taskSpecProto.getOutputSpecsCount());
    if (taskSpecProto.getOutputSpecsCount() > 0) {
        for (IOSpecProto outputSpecProto : taskSpecProto.getOutputSpecsList()) {
            outputSpecList.add(getOutputSpecFromProto(outputSpecProto));
        }
    }
    List<GroupInputSpec> groupInputSpecs = new ArrayList<GroupInputSpec>(taskSpecProto.getGroupedInputSpecsCount());
    if (taskSpecProto.getGroupedInputSpecsCount() > 0) {
        for (GroupInputSpecProto groupInputSpecProto : taskSpecProto.getGroupedInputSpecsList()) {
            groupInputSpecs.add(getGroupInputSpecFromProto(groupInputSpecProto));
        }
    }
    Configuration taskConf = null;
    if (taskSpecProto.hasTaskConf()) {
        taskConf = new Configuration(false);
        Map<String, String> confMap = DagTypeConverters.convertConfFromProto(taskSpecProto.getTaskConf());
        for (Entry<String, String> e : confMap.entrySet()) {
            taskConf.set(e.getKey(), e.getValue());
        }
    }
    TaskSpec taskSpec = new TaskSpec(taskAttemptID, taskSpecProto.getDagName(), taskSpecProto.getVertexName(), taskSpecProto.getVertexParallelism(), processorDescriptor, inputSpecList, outputSpecList, groupInputSpecs, taskConf);
    return taskSpec;
}

Example 23 with InputSpec

use of org.apache.tez.runtime.api.impl.InputSpec in project tez by apache.

the class LogicalIOProcessorRuntimeTask method close.

public void close() throws Exception {
    try {
        Preconditions.checkState(this.state.get() == State.RUNNING, "Can only run while in RUNNING state. Current: " + this.state);
        this.state.set(State.CLOSED);
        // Close the Inputs.
        for (InputSpec inputSpec : inputSpecs) {
            String srcVertexName = inputSpec.getSourceVertexName();
            initializedInputs.remove(srcVertexName);
            List<Event> closeInputEvents = ((InputFrameworkInterface) inputsMap.get(srcVertexName)).close();
            sendTaskGeneratedEvents(closeInputEvents, EventProducerConsumerType.INPUT, taskSpec.getVertexName(), srcVertexName, taskSpec.getTaskAttemptID());
        }
        // Close the Outputs.
        for (OutputSpec outputSpec : outputSpecs) {
            String destVertexName = outputSpec.getDestinationVertexName();
            initializedOutputs.remove(destVertexName);
            List<Event> closeOutputEvents = ((LogicalOutputFrameworkInterface) outputsMap.get(destVertexName)).close();
            sendTaskGeneratedEvents(closeOutputEvents, EventProducerConsumerType.OUTPUT, taskSpec.getVertexName(), destVertexName, taskSpec.getTaskAttemptID());
        }
        // Close the Processor.
        processorClosed = true;
        processor.close();
    } finally {
        setTaskDone();
        // Clear the interrupt status since the task execution is done.
        Thread.interrupted();
        if (eventRouterThread != null) {
            eventRouterThread.interrupt();
            LOG.info("Joining on EventRouter");
            try {
                eventRouterThread.join();
            } catch (InterruptedException e) {
                LOG.info("Ignoring interrupt while waiting for the router thread to die");
                Thread.currentThread().interrupt();
            }
            eventRouterThread = null;
        }
        String timeStamp = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime());
        System.err.println(timeStamp + " Completed running task attempt: " + taskSpec.getTaskAttemptID().toString());
        System.out.println(timeStamp + " Completed running task attempt: " + taskSpec.getTaskAttemptID().toString());
    }
}

Example 24 with InputSpec

use of org.apache.tez.runtime.api.impl.InputSpec in project tez by apache.

the class TestTezTaskRunner2 method testTaskConfUsage.

@Test(timeout = 5000)
public void testTaskConfUsage() throws Exception {
    Configuration conf = new Configuration(false);
    conf.set("global", "global1");
    conf.set("global_override", "global1");
    String[] localDirs = null;
    Configuration taskConf = new Configuration(false);
    conf.set("global_override", "task1");
    conf.set("task", "task1");
    List<InputSpec> inputSpecList = new ArrayList<>();
    List<OutputSpec> outputSpecList = new ArrayList<>();
    TaskSpec taskSpec = new TaskSpec("dagName", "vertexName", 1, mock(ProcessorDescriptor.class), inputSpecList, outputSpecList, null, taskConf);
    TezExecutors sharedExecutor = new TezSharedExecutor(conf);
    TezTaskRunner2 taskRunner2 = new TezTaskRunner2(conf, mock(UserGroupInformation.class), localDirs, taskSpec, 1, null, null, null, mock(TaskReporter.class), null, null, "pid", null, 1000, false, new DefaultHadoopShim(), sharedExecutor);
    Assert.assertEquals("global1", taskRunner2.task.getTaskConf().get("global"));
    Assert.assertEquals("task1", taskRunner2.task.getTaskConf().get("global_override"));
    Assert.assertEquals("task1", taskRunner2.task.getTaskConf().get("task"));
    sharedExecutor.shutdownNow();
}

Example 25 with InputSpec

use of org.apache.tez.runtime.api.impl.InputSpec in project tez by apache.

the class TestLogicalIOProcessorRuntimeTask method createInputSpecList.

private List<InputSpec> createInputSpecList() {
    InputDescriptor inputDesc = InputDescriptor.create(TestInput.class.getName());
    InputSpec inputSpec = new InputSpec("inedge", inputDesc, 1);
    return Lists.newArrayList(inputSpec);
}