Examples with Task org.apache.tez.dag.app.dag.Task used on opensource projects

Example 26 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class AMWebController method getRequestedTasks.

/**
 * getRequestedTasks
 * Heart of getTasksInfo. Given a dag and a limit, based on the incoming query parameters
 * returns a list of task instances
 *
 * @param dag {DAG}
 * @param limit {Integer}
 */
List<Task> getRequestedTasks(DAG dag, Integer limit) {
    List<Task> tasks = new ArrayList<Task>();
    List<List<Integer>> taskIDs = getIDsFromRequest(WebUIService.TASK_ID, limit, 2);
    if (taskIDs == null) {
        return null;
    } else if (!taskIDs.isEmpty()) {
        for (List<Integer> indexes : taskIDs) {
            Vertex vertex = getVertexFromIndex(dag, indexes.get(0));
            if (vertex == null) {
                continue;
            }
            Task task = vertex.getTask(indexes.get(1));
            if (task == null) {
                continue;
            } else {
                tasks.add(task);
            }
            if (tasks.size() >= limit) {
                break;
            }
        }
    } else {
        List<Integer> vertexIDs = getIntegersFromRequest(WebUIService.VERTEX_ID, limit);
        if (vertexIDs == null) {
            return null;
        } else if (!vertexIDs.isEmpty()) {
            for (Integer vertexID : vertexIDs) {
                Vertex vertex = getVertexFromIndex(dag, vertexID);
                if (vertex == null) {
                    continue;
                }
                List<Task> vertexTasks = new ArrayList<Task>(vertex.getTasks().values());
                tasks.addAll(vertexTasks.subList(0, Math.min(vertexTasks.size(), limit - tasks.size())));
                if (tasks.size() >= limit) {
                    break;
                }
            }
        } else {
            Collection<Vertex> vertices = dag.getVertices().values();
            for (Vertex vertex : vertices) {
                List<Task> vertexTasks = new ArrayList<Task>(vertex.getTasks().values());
                tasks.addAll(vertexTasks.subList(0, Math.min(vertexTasks.size(), limit - tasks.size())));
                if (tasks.size() >= limit) {
                    break;
                }
            }
        }
    }
    return tasks;
}

Example 27 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class TestVertexImpl method testInputInitializerEventsMultipleSources.

@Test(timeout = 10000)
public void testInputInitializerEventsMultipleSources() throws Exception {
    useCustomInitializer = true;
    customInitializer = new EventHandlingRootInputInitializer(null);
    EventHandlingRootInputInitializer initializer = (EventHandlingRootInputInitializer) customInitializer;
    initializer.setNumExpectedEvents(4);
    setupPreDagCreation();
    dagPlan = createDAGPlanWithRunningInitializer4();
    setupPostDagCreation();
    VertexImplWithRunningInputInitializer v1 = (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
    VertexImplWithRunningInputInitializer v2 = (VertexImplWithRunningInputInitializer) vertices.get("vertex2");
    VertexImplWithRunningInputInitializer v3 = (VertexImplWithRunningInputInitializer) vertices.get("vertex3");
    initVertex(v1);
    startVertex(v1);
    dispatcher.await();
    // Vertex1 start should trigger downstream vertices
    Assert.assertEquals(VertexState.RUNNING, v1.getState());
    Assert.assertEquals(VertexState.RUNNING, v2.getState());
    Assert.assertEquals(VertexState.INITIALIZING, v3.getState());
    List<ByteBuffer> expectedPayloads = new LinkedList<ByteBuffer>();
    // Genrate events from v1 to v3's InputInitializer
    ByteBuffer payload = ByteBuffer.allocate(12).putInt(0, 1).putInt(4, 0).putInt(8, 0);
    expectedPayloads.add(payload);
    InputInitializerEvent event = InputInitializerEvent.create("vertex3", "input1", payload);
    // Create taskId and taskAttemptId for the single task that exists in vertex1
    TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
    TezEvent tezEvent = new TezEvent(event, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", ta0_t0_v1));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
    dispatcher.await();
    // Events should not be cached in the vertex, since the initializer is running
    Assert.assertEquals(0, v3.pendingInitializerEvents.size());
    // Events should be cached since the tasks have not succeeded.
    // Verify that events are cached
    RootInputInitializerManager.InitializerWrapper initializerWrapper = v3.rootInputInitializerManager.getInitializerWrapper("input1");
    Assert.assertEquals(1, initializerWrapper.getFirstSuccessfulAttemptMap().size());
    Assert.assertEquals(1, initializerWrapper.getPendingEvents().get(v1.getName()).size());
    // Get all tasks of vertex1 to succeed.
    for (TezTaskID taskId : v1.getTasks().keySet()) {
        TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0);
        v1.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED));
        v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
        dispatcher.await();
        v1.stateChangeNotifier.taskSucceeded(v1.getName(), taskId, taskAttemptId.getId());
    }
    dispatcher.await();
    Assert.assertEquals(1, initializer.initializerEvents.size());
    // Test written based on this
    Assert.assertEquals(2, v2.getTotalTasks());
    // Generate events from v2 to v3's initializer. 1 from task 0, 2 from task 1
    for (Task task : v2.getTasks().values()) {
        TezTaskID taskId = task.getTaskId();
        TezTaskAttemptID attemptId = TezTaskAttemptID.getInstance(taskId, 0);
        int numEventsFromTask = taskId.getId() + 1;
        for (int i = 0; i < numEventsFromTask; i++) {
            payload = ByteBuffer.allocate(12).putInt(0, 2).putInt(4, taskId.getId()).putInt(8, i);
            expectedPayloads.add(payload);
            InputInitializerEvent event2 = InputInitializerEvent.create("vertex3", "input1", payload);
            TezEvent tezEvent2 = new TezEvent(event2, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", attemptId));
            dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Collections.singletonList(tezEvent2)));
            dispatcher.await();
        }
    }
    // Validate queueing of these events
    // Only v2 events pending
    Assert.assertEquals(1, initializerWrapper.getPendingEvents().keySet().size());
    // 3 events pending
    Assert.assertEquals(3, initializerWrapper.getPendingEvents().get(v2.getName()).size());
    // Get all tasks of vertex1 to succeed.
    for (TezTaskID taskId : v2.getTasks().keySet()) {
        TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0);
        v2.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED));
        v2.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
        dispatcher.await();
        v2.stateChangeNotifier.taskSucceeded(v2.getName(), taskId, taskAttemptId.getId());
    }
    dispatcher.await();
    // Initializer would have run, and processed events.
    while (v3.getState() != VertexState.RUNNING) {
        Thread.sleep(10);
    }
    Assert.assertEquals(VertexState.RUNNING, v3.getState());
    Assert.assertEquals(4, initializer.initializerEvents.size());
    Assert.assertTrue(initializer.initComplete.get());
    Assert.assertEquals(2, initializerWrapper.getFirstSuccessfulAttemptMap().size());
    Assert.assertEquals(0, initializerWrapper.getPendingEvents().get(v1.getName()).size());
    for (InputInitializerEvent initializerEvent : initializer.initializerEvents) {
        expectedPayloads.remove(initializerEvent.getUserPayload());
    }
    Assert.assertEquals(0, expectedPayloads.size());
}

Example 28 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class TestVertexImpl method testCounterLimits.

@Test(timeout = 5000)
public void testCounterLimits() {
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex2");
    startVertex(v);
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
    for (int i = 0; i < 2; ++i) {
        TezCounters ctrs = new TezCounters();
        for (int j = 0; j < 75; ++j) {
            ctrs.findCounter("g", "c" + i + "_" + j).increment(1);
        }
        Task t = v.getTask(i);
        ((TaskImpl) t).setCounters(ctrs);
    }
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, v.getState());
    Assert.assertEquals(1, v.getCompletedTasks());
    Assert.assertTrue((0.5f) == v.getCompletedTaskProgress());
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v.getState());
    Assert.assertEquals(2, v.getCompletedTasks());
    System.out.println(v.getDiagnostics());
    Assert.assertTrue("Diagnostics should contain counter limits error message", StringUtils.join(v.getDiagnostics(), ",").contains("Counters limit exceeded"));
}

Example 29 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class TestLocalTaskSchedulerService method testDeallocationAfterAllocation.

/**
 * TaskAttempt Killed from START_WAIT
 */
@Test(timeout = 5000)
public void testDeallocationAfterAllocation() throws InterruptedException {
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(10000l, 1), 1);
    TaskSchedulerContext mockContext = TestTaskSchedulerHelpers.setupMockTaskSchedulerContext("", 0, "", false, appAttemptId, 10000l, null, new Configuration());
    MockLocalTaskSchedulerSerivce taskSchedulerService = new MockLocalTaskSchedulerSerivce(mockContext);
    taskSchedulerService.initialize();
    taskSchedulerService.start();
    Task task = mock(Task.class);
    taskSchedulerService.allocateTask(task, Resource.newInstance(1024, 1), null, null, Priority.newInstance(1), null, null);
    taskSchedulerService.startRequestHandlerThread();
    MockAsyncDelegateRequestHandler requestHandler = taskSchedulerService.getRequestHandler();
    requestHandler.drainRequest(1);
    taskSchedulerService.deallocateTask(task, false, null, null);
    requestHandler.drainRequest(2);
    assertEquals(1, requestHandler.deallocateCount);
    assertEquals(1, requestHandler.allocateCount);
    taskSchedulerService.shutdown();
}

Example 30 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class TestDAGImpl method testEdgeManager_RouteDataMovementEventToDestination.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testEdgeManager_RouteDataMovementEventToDestination() {
    setupDAGWithCustomEdge(ExceptionLocation.RouteDataMovementEventToDestination);
    dispatcher.getEventHandler().handle(new DAGEvent(dagWithCustomEdge.getID(), DAGEventType.DAG_INIT));
    dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), null));
    dispatcher.await();
    Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState());
    VertexImpl v1 = (VertexImpl) dagWithCustomEdge.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dagWithCustomEdge.getVertex("vertex2");
    dispatcher.await();
    Task t1 = v2.getTask(0);
    TaskAttemptImpl ta1 = (TaskAttemptImpl) t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskId(), 0));
    DataMovementEvent daEvent = DataMovementEvent.create(ByteBuffer.wrap(new byte[0]));
    TezEvent tezEvent = new TezEvent(daEvent, new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", "vertex2", ta1.getID()));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent)));
    dispatcher.await();
    v2.getTaskAttemptTezEvents(ta1.getID(), 0, 0, 1000);
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v2.getState());
    Assert.assertEquals(VertexState.KILLED, v1.getState());
    String diag = StringUtils.join(v2.getDiagnostics(), ",");
    Assert.assertTrue(diag.contains(ExceptionLocation.RouteDataMovementEventToDestination.name()));
}