Examples with InputSpec org.apache.tez.runtime.api.impl.InputSpec used on opensource projects

Example 1 with InputSpec

use of org.apache.tez.runtime.api.impl.InputSpec in project tez by apache.

the class VertexImpl method getInputSpecList.

@Override
public List<InputSpec> getInputSpecList(int taskIndex) throws AMUserCodeException {
    // For locking strategy, please refer to getOutputSpecList()
    readLock.lock();
    List<InputSpec> inputSpecList = null;
    try {
        inputSpecList = new ArrayList<InputSpec>(this.getInputVerticesCount() + (rootInputDescriptors == null ? 0 : rootInputDescriptors.size()));
        if (rootInputDescriptors != null) {
            for (Entry<String, RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> rootInputDescriptorEntry : rootInputDescriptors.entrySet()) {
                inputSpecList.add(new InputSpec(rootInputDescriptorEntry.getKey(), rootInputDescriptorEntry.getValue().getIODescriptor(), rootInputSpecs.get(rootInputDescriptorEntry.getKey()).getNumPhysicalInputsForWorkUnit(taskIndex)));
            }
        }
    } finally {
        readLock.unlock();
    }
    for (Vertex vertex : getInputVertices().keySet()) {
        /**
         * It is possible that setParallelism is in the middle of processing in target vertex with
         * its write lock. So we need to get inputspec by acquiring read lock in target vertex to
         * get consistent view.
         * Refer TEZ-2251
         */
        InputSpec inputSpec = ((VertexImpl) vertex).getDestinationSpecFor(this, taskIndex);
        // TODO DAGAM This should be based on the edge type.
        inputSpecList.add(inputSpec);
    }
    return inputSpecList;
}

Example 2 with InputSpec

use of org.apache.tez.runtime.api.impl.InputSpec in project tez by apache.

the class TestVertexImpl method testVertexRootInputSpecUpdateAll.

@Test(timeout = 5000)
public void testVertexRootInputSpecUpdateAll() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithInputInitializer("TestInputInitializer");
    setupPostDagCreation();
    int expectedNumTasks = RootInputSpecUpdaterVertexManager.NUM_TASKS;
    VertexImplWithControlledInitializerManager v3 = (VertexImplWithControlledInitializerManager) vertices.get("vertex3");
    dispatcher.getEventHandler().handle(new VertexEvent(v3.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, v3.getState());
    RootInputInitializerManagerControlled initializerManager1 = v3.getRootInputInitializerManager();
    initializerManager1.completeInputInitialization();
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v3.getState());
    Assert.assertEquals(expectedNumTasks, v3.getTotalTasks());
    Assert.assertEquals(RootInputSpecUpdaterVertexManager.class.getName(), v3.getVertexManager().getPlugin().getClass().getName());
    Assert.assertEquals(true, initializerManager1.hasShutDown);
    for (int i = 0; i < expectedNumTasks; i++) {
        List<InputSpec> inputSpecs = v3.getInputSpecList(i);
        Assert.assertEquals(1, inputSpecs.size());
        Assert.assertEquals(4, inputSpecs.get(0).getPhysicalEdgeCount());
    }
}

Example 3 with InputSpec

use of org.apache.tez.runtime.api.impl.InputSpec in project tez by apache.

the class TestVertexImpl method testVertexWithInitializerSuccessLegacyRouting.

@Test(timeout = 5000)
public void testVertexWithInitializerSuccessLegacyRouting() throws Exception {
    // Remove after legacy routing is removed
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithInputInitializer("TestInputInitializer");
    setupPostDagCreation();
    VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices.get("vertex1");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
    RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
    List<TaskLocationHint> v1Hints = createTaskLocationHints(5);
    initializerManager1.completeInputInitialization(0, 5, v1Hints);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v1.getState());
    Assert.assertEquals(5, v1.getTotalTasks());
    Assert.assertEquals(RootInputVertexManager.class.getName(), v1.getVertexManager().getPlugin().getClass().getName());
    for (int i = 0; i < v1Hints.size(); ++i) {
        Assert.assertEquals(v1Hints.get(i), v1.getTaskLocationHints()[i]);
    }
    Assert.assertEquals(true, initializerManager1.hasShutDown);
    for (int i = 0; i < 5; i++) {
        List<InputSpec> inputSpecs = v1.getInputSpecList(i);
        Assert.assertEquals(1, inputSpecs.size());
        Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount());
    }
    // task events get buffered
    Assert.assertEquals(5, v1.pendingTaskEvents.size());
    VertexImplWithControlledInitializerManager v2 = (VertexImplWithControlledInitializerManager) vertices.get("vertex2");
    Assert.assertEquals(VertexState.INITIALIZING, v2.getState());
    // non-task events don't get buffered
    List<TezEvent> events = Lists.newLinkedList();
    TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
    events.add(new TezEvent(VertexManagerEvent.create("vertex2", ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.PROCESSOR, "vertex1", "vertex2", ta0_t0_v1)));
    events.add(new TezEvent(InputDataInformationEvent.createWithSerializedPayload(0, ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.INPUT, "vertex2", "NULL_VERTEX", null)));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), events));
    dispatcher.await();
    Assert.assertEquals(1, v2.pendingTaskEvents.size());
    RootInputInitializerManagerControlled initializerManager2 = v2.getRootInputInitializerManager();
    List<TaskLocationHint> v2Hints = createTaskLocationHints(10);
    initializerManager2.completeInputInitialization(0, 10, v2Hints);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v2.getState());
    Assert.assertEquals(10, v2.getTotalTasks());
    Assert.assertEquals(RootInputVertexManager.class.getName(), v2.getVertexManager().getPlugin().getClass().getName());
    for (int i = 0; i < v2Hints.size(); ++i) {
        Assert.assertEquals(v2Hints.get(i), v2.getTaskLocationHints()[i]);
    }
    Assert.assertEquals(true, initializerManager2.hasShutDown);
    // task events get buffered
    Assert.assertEquals(11, v2.pendingTaskEvents.size());
    for (int i = 0; i < 10; i++) {
        List<InputSpec> inputSpecs = v1.getInputSpecList(i);
        Assert.assertEquals(1, inputSpecs.size());
        Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount());
    }
}

Example 4 with InputSpec

use of org.apache.tez.runtime.api.impl.InputSpec in project tez by apache.

the class TestVertexImpl method testFailuresMaxPercentSourceTaskAttemptCompletionEvents.

@Test(timeout = 5000)
public void testFailuresMaxPercentSourceTaskAttemptCompletionEvents() throws TezException {
    LOG.info("Testing testFailuresMaxPercentSourceTaskAttemptCompletionEvents");
    // Override the basic setup for this test to inject the specific config setting needed for this test
    useCustomInitializer = false;
    customInitializer = null;
    setupPreDagCreation();
    conf.setFloat(TezConfiguration.TEZ_VERTEX_FAILURES_MAXPERCENT, 50.0f);
    conf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1);
    dagPlan = createTestDAGPlan();
    setupPostDagCreation();
    initAllVertices(VertexState.INITED);
    VertexImpl v4 = vertices.get("vertex4");
    VertexImpl v5 = vertices.get("vertex5");
    VertexImpl v6 = vertices.get("vertex6");
    startVertex(vertices.get("vertex1"));
    startVertex(vertices.get("vertex2"));
    dispatcher.await();
    LOG.info("Verifying v6 state " + v6.getState());
    Assert.assertEquals(VertexState.RUNNING, v6.getState());
    TezTaskID t1_v4 = TezTaskID.getInstance(v4.getVertexId(), 0);
    TezTaskID t2_v4 = TezTaskID.getInstance(v4.getVertexId(), 1);
    TezTaskID t1_v5 = TezTaskID.getInstance(v5.getVertexId(), 0);
    TezTaskID t2_v5 = TezTaskID.getInstance(v5.getVertexId(), 1);
    TezTaskAttemptID ta1_t1_v4 = TezTaskAttemptID.getInstance(t1_v4, 0);
    TezTaskAttemptID ta1_t2_v4 = TezTaskAttemptID.getInstance(t2_v4, 0);
    TezTaskAttemptID ta1_t1_v5 = TezTaskAttemptID.getInstance(t1_v5, 0);
    TezTaskAttemptID ta1_t2_v5 = TezTaskAttemptID.getInstance(t2_v5, 0);
    TaskSpec taskSpec = new TaskSpec("dag", "vertex", 2, new ProcessorDescriptor(), new ArrayList<InputSpec>(), new ArrayList<OutputSpec>(), null, conf);
    TaskLocationHint locationHint = TaskLocationHint.createTaskLocationHint(null, null);
    // Tasks can only succeed from a scheduled or running state
    dispatcher.getEventHandler().handle(new TaskEventScheduleTask(t1_v4, taskSpec, locationHint, false));
    dispatcher.getEventHandler().handle(new TaskEventScheduleTask(t2_v4, taskSpec, locationHint, false));
    // Completed tasks are less that the max percent failure
    dispatcher.getEventHandler().handle(new TaskEventTAFailed(ta1_t1_v4, TaskFailureType.NON_FATAL, null));
    dispatcher.getEventHandler().handle(new TaskEventTASucceeded(ta1_t2_v4));
    dispatcher.getEventHandler().handle(new TaskEventTASucceeded(ta1_t1_v5));
    dispatcher.getEventHandler().handle(new TaskEventTAFailed(ta1_t2_v5, TaskFailureType.NON_FATAL, null));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v4.getState());
    Assert.assertEquals(VertexState.SUCCEEDED, v5.getState());
    Assert.assertEquals(VertexState.RUNNING, v6.getState());
    Assert.assertEquals(4, v6.numSuccessSourceAttemptCompletions);
}

Example 5 with InputSpec

use of org.apache.tez.runtime.api.impl.InputSpec in project tez by apache.

the class TestVertexImpl method testVertexWithInitializerSuccess.

@Test(timeout = 500000)
public void testVertexWithInitializerSuccess() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithInputInitializer("TestInputInitializer");
    setupPostDagCreation();
    VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices.get("vertex1");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
    RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
    List<TaskLocationHint> v1Hints = createTaskLocationHints(5);
    initializerManager1.completeInputInitialization(0, 5, v1Hints);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v1.getState());
    Assert.assertEquals(5, v1.getTotalTasks());
    Assert.assertEquals(RootInputVertexManager.class.getName(), v1.getVertexManager().getPlugin().getClass().getName());
    for (int i = 0; i < v1Hints.size(); ++i) {
        Assert.assertEquals(v1Hints.get(i), v1.getTaskLocationHints()[i]);
    }
    Assert.assertEquals(true, initializerManager1.hasShutDown);
    for (int i = 0; i < 5; i++) {
        List<InputSpec> inputSpecs = v1.getInputSpecList(i);
        Assert.assertEquals(1, inputSpecs.size());
        Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount());
    }
    List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
    // scheduling start to trigger edge routing to begin
    for (int i = 0; i < v1.getTotalTasks(); ++i) {
        taskList.add(ScheduleTaskRequest.create(i, null));
    }
    v1.scheduleTasks(taskList);
    dispatcher.await();
    // check all tasks get their events
    for (int i = 0; i < v1.getTotalTasks(); ++i) {
        Assert.assertEquals(1, v1.getTaskAttemptTezEvents(TezTaskAttemptID.getInstance(v1.getTask(i).getTaskId(), 0), 0, 0, 100).getEvents().size());
    }
    VertexImplWithControlledInitializerManager v2 = (VertexImplWithControlledInitializerManager) vertices.get("vertex2");
    Assert.assertEquals(VertexState.INITIALIZING, v2.getState());
    // non-task events don't get buffered
    List<TezEvent> events = Lists.newLinkedList();
    TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
    events.add(new TezEvent(VertexManagerEvent.create("vertex2", ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.PROCESSOR, "vertex1", "vertex2", ta0_t0_v1)));
    events.add(new TezEvent(InputDataInformationEvent.createWithSerializedPayload(0, ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.INPUT, "vertex2", "NULL_VERTEX", null)));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), events));
    dispatcher.await();
    RootInputInitializerManagerControlled initializerManager2 = v2.getRootInputInitializerManager();
    List<TaskLocationHint> v2Hints = createTaskLocationHints(10);
    initializerManager2.completeInputInitialization(0, 10, v2Hints);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v2.getState());
    Assert.assertEquals(10, v2.getTotalTasks());
    Assert.assertEquals(RootInputVertexManager.class.getName(), v2.getVertexManager().getPlugin().getClass().getName());
    for (int i = 0; i < v2Hints.size(); ++i) {
        Assert.assertEquals(v2Hints.get(i), v2.getTaskLocationHints()[i]);
    }
    Assert.assertEquals(true, initializerManager2.hasShutDown);
    // scheduling start to trigger edge routing to begin
    taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
    // scheduling start to trigger edge routing to begin
    for (int i = 0; i < v2.getTotalTasks(); ++i) {
        taskList.add(ScheduleTaskRequest.create(i, null));
    }
    v2.scheduleTasks(taskList);
    dispatcher.await();
    // check all tasks get their events
    for (int i = 0; i < v2.getTotalTasks(); ++i) {
        Assert.assertEquals(((i == 0) ? 2 : 1), v2.getTaskAttemptTezEvents(TezTaskAttemptID.getInstance(v2.getTask(i).getTaskId(), 0), 0, 0, 100).getEvents().size());
    }
    for (int i = 0; i < 10; i++) {
        List<InputSpec> inputSpecs = v1.getInputSpecList(i);
        Assert.assertEquals(1, inputSpecs.size());
        Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount());
    }
}