Examples with TezVertexID org.apache.tez.dag.records.TezVertexID used on opensource projects

Example 71 with TezVertexID

use of org.apache.tez.dag.records.TezVertexID in project hive by apache.

the class TaskExecutorTestHelpers method createSubmitWorkRequestProto.

public static SubmitWorkRequestProto createSubmitWorkRequestProto(int fragmentNumber, int selfAndUpstreamParallelism, int selfAndUpstreamComplete, long firstAttemptStartTime, long currentAttemptStartTime, int withinDagPriority, String dagName, int dagId, boolean isGuaranteed) {
    ApplicationId appId = ApplicationId.newInstance(9999, 72);
    TezDAGID dag = TezDAGID.getInstance(appId, 1);
    TezVertexID vId = TezVertexID.getInstance(dag, dagId);
    return SubmitWorkRequestProto.newBuilder().setAttemptNumber(0).setFragmentNumber(fragmentNumber).setWorkSpec(VertexOrBinary.newBuilder().setVertex(SignableVertexSpec.newBuilder().setDagName(dagName).setHiveQueryId(dagName).setUser("MockUser").setTokenIdentifier("MockToken_1").setQueryIdentifier(QueryIdentifierProto.newBuilder().setApplicationIdString(appId.toString()).setAppAttemptNumber(0).setDagIndex(dag.getId()).build()).setVertexIndex(vId.getId()).setVertexName("MockVertex").setProcessorDescriptor(LlapDaemonProtocolProtos.EntityDescriptorProto.newBuilder().setClassName("MockProcessor").build()).build()).build()).setAmHost("localhost").setAmPort(12345).setContainerIdString("MockContainer_1").setIsGuaranteed(isGuaranteed).setFragmentRuntimeInfo(LlapDaemonProtocolProtos.FragmentRuntimeInfo.newBuilder().setFirstAttemptStartTime(firstAttemptStartTime).setCurrentAttemptStartTime(currentAttemptStartTime).setNumSelfAndUpstreamTasks(selfAndUpstreamParallelism).setNumSelfAndUpstreamCompletedTasks(selfAndUpstreamComplete).setWithinDagPriority(withinDagPriority).build()).build();
}

Example 72 with TezVertexID

use of org.apache.tez.dag.records.TezVertexID in project hive by apache.

the class TaskSpecBuilder method constructTaskSpec.

public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) {
    Vertex vertex = dag.getVertex(vertexName);
    ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor();
    List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs();
    List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs();
    Preconditions.checkState(inputs.size() == 1);
    Preconditions.checkState(outputs.size() == 1);
    List<InputSpec> inputSpecs = new ArrayList<>();
    for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) {
        InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1);
        inputSpecs.add(inputSpec);
    }
    List<OutputSpec> outputSpecs = new ArrayList<>();
    for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) {
        OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1);
        outputSpecs.add(outputSpec);
    }
    TezDAGID dagId = TezDAGID.getInstance(appId, 0);
    TezVertexID vertexId = TezVertexID.getInstance(dagId, 0);
    TezTaskID taskId = TezTaskID.getInstance(vertexId, index);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0);
    return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null);
}

Example 73 with TezVertexID

use of org.apache.tez.dag.records.TezVertexID in project hive by apache.

the class TaskExecutorTestHelpers method createSubmitWorkRequestProto.

public static SubmitWorkRequestProto createSubmitWorkRequestProto(int fragmentNumber, int selfAndUpstreamParallelism, int selfAndUpstreamComplete, long firstAttemptStartTime, long currentAttemptStartTime, int withinDagPriority, String dagName) {
    ApplicationId appId = ApplicationId.newInstance(9999, 72);
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vId = TezVertexID.getInstance(dagId, 35);
    return SubmitWorkRequestProto.newBuilder().setAttemptNumber(0).setFragmentNumber(fragmentNumber).setWorkSpec(VertexOrBinary.newBuilder().setVertex(SignableVertexSpec.newBuilder().setDagName(dagName).setHiveQueryId(dagName).setUser("MockUser").setTokenIdentifier("MockToken_1").setQueryIdentifier(QueryIdentifierProto.newBuilder().setApplicationIdString(appId.toString()).setAppAttemptNumber(0).setDagIndex(dagId.getId()).build()).setVertexIndex(vId.getId()).setVertexName("MockVertex").setProcessorDescriptor(LlapDaemonProtocolProtos.EntityDescriptorProto.newBuilder().setClassName("MockProcessor").build()).build()).build()).setAmHost("localhost").setAmPort(12345).setContainerIdString("MockContainer_1").setFragmentRuntimeInfo(LlapDaemonProtocolProtos.FragmentRuntimeInfo.newBuilder().setFirstAttemptStartTime(firstAttemptStartTime).setCurrentAttemptStartTime(currentAttemptStartTime).setNumSelfAndUpstreamTasks(selfAndUpstreamParallelism).setNumSelfAndUpstreamCompletedTasks(selfAndUpstreamComplete).setWithinDagPriority(withinDagPriority).build()).build();
}

Example 74 with TezVertexID

use of org.apache.tez.dag.records.TezVertexID in project tez by apache.

the class TestVertexImpl method setupVertices.

private void setupVertices() {
    int vCnt = dagPlan.getVertexCount();
    LOG.info("Setting up vertices from dag plan, verticesCnt=" + vCnt);
    vertices = new HashMap<String, VertexImpl>();
    vertexIdMap = new HashMap<TezVertexID, VertexImpl>();
    Configuration dagConf = new Configuration(false);
    dagConf.set("abc", "foobar");
    for (int i = 0; i < vCnt; ++i) {
        VertexPlan vPlan = dagPlan.getVertex(i);
        String vName = vPlan.getName();
        TezVertexID vertexId = TezVertexID.getInstance(dagId, i + 1);
        VertexImpl v = null;
        VertexLocationHint locationHint = DagTypeConverters.convertFromDAGPlan(vPlan.getTaskLocationHintList());
        if (useCustomInitializer) {
            if (customInitializer == null) {
                v = new VertexImplWithControlledInitializerManager(vertexId, vPlan, vPlan.getName(), conf, dispatcher.getEventHandler(), taskCommunicatorManagerInterface, clock, thh, appContext, locationHint, dispatcher, updateTracker, dagConf);
            } else {
                v = new VertexImplWithRunningInputInitializer(vertexId, vPlan, vPlan.getName(), conf, dispatcher.getEventHandler(), taskCommunicatorManagerInterface, clock, thh, appContext, locationHint, dispatcher, customInitializer, updateTracker, dagConf);
            }
        } else {
            v = new VertexImpl(vertexId, vPlan, vPlan.getName(), conf, dispatcher.getEventHandler(), taskCommunicatorManagerInterface, clock, thh, true, appContext, locationHint, vertexGroups, taskSpecificLaunchCmdOption, updateTracker, dagConf);
        }
        vertices.put(vName, v);
        vertexIdMap.put(vertexId, v);
    }
}

Example 75 with TezVertexID

use of org.apache.tez.dag.records.TezVertexID in project tez by apache.

the class TestSpeculation method testBasicSpeculationPerVertexConf.

@Test(timeout = 10000)
public void testBasicSpeculationPerVertexConf() throws Exception {
    DAG dag = DAG.create("test");
    String vNameNoSpec = "A";
    String vNameSpec = "B";
    Vertex vA = Vertex.create(vNameNoSpec, ProcessorDescriptor.create("Proc.class"), 5);
    Vertex vB = Vertex.create(vNameSpec, ProcessorDescriptor.create("Proc.class"), 5);
    vA.setConf(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, "false");
    dag.addVertex(vA);
    dag.addVertex(vB);
    // min/max src fraction is set to 1. So vertices will run sequentially
    dag.addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("O"), InputDescriptor.create("I"))));
    MockTezClient tezClient = createTezSession();
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    TezVertexID vertexId = dagImpl.getVertex(vNameSpec).getVertexId();
    TezVertexID vertexIdNoSpec = dagImpl.getVertex(vNameNoSpec).getVertexId();
    // original attempt is killed and speculative one is successful
    TezTaskAttemptID killedTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    TezTaskAttemptID noSpecTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexIdNoSpec, 0), 0);
    // cause speculation trigger for both
    mockLauncher.setStatusUpdatesForTask(killedTaId, 100);
    mockLauncher.setStatusUpdatesForTask(noSpecTaId, 100);
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    org.apache.tez.dag.app.dag.Vertex vSpec = dagImpl.getVertex(vertexId);
    org.apache.tez.dag.app.dag.Vertex vNoSpec = dagImpl.getVertex(vertexIdNoSpec);
    // speculation for vA but not for vB
    Assert.assertTrue(vSpec.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue() > 0);
    Assert.assertEquals(0, vNoSpec.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
    tezClient.stop();
}