Examples with CallableEvent org.apache.tez.dag.app.dag.event.CallableEvent used on opensource projects

Example 1 with CallableEvent

use of org.apache.tez.dag.app.dag.event.CallableEvent in project tez by apache.

the class VertexImpl method commitOrFinish.

private static VertexState commitOrFinish(final VertexImpl vertex) {
    // commit only once. Dont commit shared outputs
    if (vertex.outputCommitters != null && !vertex.outputCommitters.isEmpty()) {
        if (vertex.recoveryData != null && vertex.recoveryData.isVertexCommitted()) {
            LOG.info("Vertex was already committed as per recovery" + " data, vertex=" + vertex.logIdentifier);
            return vertex.finished(VertexState.SUCCEEDED);
        }
        boolean firstCommit = true;
        for (Entry<String, OutputCommitter> entry : vertex.outputCommitters.entrySet()) {
            final OutputCommitter committer = entry.getValue();
            final String outputName = entry.getKey();
            if (vertex.sharedOutputs.contains(outputName)) {
                // dont commit shared committers. Will be committed by the DAG
                continue;
            }
            if (firstCommit) {
                LOG.info("Invoking committer commit for vertex, vertexId=" + vertex.logIdentifier);
                // Log commit start event on first actual commit
                try {
                    vertex.appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(vertex.getDAGId(), new VertexCommitStartedEvent(vertex.vertexId, vertex.clock.getTime())));
                } catch (IOException e) {
                    LOG.error("Failed to persist commit start event to recovery, vertex=" + vertex.logIdentifier, e);
                    vertex.trySetTerminationCause(VertexTerminationCause.RECOVERY_ERROR);
                    return vertex.finished(VertexState.FAILED);
                }
                firstCommit = false;
            }
            VertexCommitCallback commitCallback = new VertexCommitCallback(vertex, outputName);
            CallableEvent commitCallableEvent = new CallableEvent(commitCallback) {

                @Override
                public Void call() throws Exception {
                    try {
                        TezUtilsInternal.setHadoopCallerContext(vertex.appContext.getHadoopShim(), vertex.vertexId);
                        vertex.dagUgi.doAs(new PrivilegedExceptionAction<Void>() {

                            @Override
                            public Void run() throws Exception {
                                LOG.info("Invoking committer commit for output=" + outputName + ", vertexId=" + vertex.logIdentifier);
                                committer.commitOutput();
                                return null;
                            }
                        });
                    } finally {
                        vertex.appContext.getHadoopShim().clearHadoopCallerContext();
                    }
                    return null;
                }
            };
            ListenableFuture<Void> commitFuture = vertex.getAppContext().getExecService().submit(commitCallableEvent);
            Futures.addCallback(commitFuture, commitCallableEvent.getCallback());
            vertex.commitFutures.put(outputName, commitFuture);
        }
    }
    if (vertex.commitFutures.isEmpty()) {
        return vertex.finished(VertexState.SUCCEEDED);
    } else {
        return VertexState.COMMITTING;
    }
}

Example 2 with CallableEvent

use of org.apache.tez.dag.app.dag.event.CallableEvent in project tez by apache.

the class DAGImpl method vertexSucceeded.

private boolean vertexSucceeded(Vertex vertex) {
    numSuccessfulVertices++;
    boolean recoveryFailed = false;
    if (!commitAllOutputsOnSuccess && isCommittable()) {
        // committing successful outputs immediately. check for shared outputs
        List<VertexGroupInfo> groupsList = vertexGroupInfo.get(vertex.getName());
        if (groupsList != null) {
            List<VertexGroupInfo> commitList = Lists.newArrayListWithCapacity(groupsList.size());
            for (VertexGroupInfo groupInfo : groupsList) {
                groupInfo.successfulMembers++;
                if (groupInfo.groupMembers.size() == groupInfo.successfulMembers && !groupInfo.outputs.isEmpty()) {
                    // group has outputs and all vertex members are done
                    LOG.info("All members of group: " + groupInfo.groupName + " are succeeded. Commiting outputs");
                    commitList.add(groupInfo);
                }
            }
            for (VertexGroupInfo groupInfo : commitList) {
                if (recoveryData != null && recoveryData.isVertexGroupCommitted(groupInfo.groupName)) {
                    LOG.info("VertexGroup was already committed as per recovery" + " data, groupName=" + groupInfo.groupName);
                    for (String vertexName : groupInfo.groupMembers) {
                        VertexRecoveryData vertexRecoveryData = recoveryData.getVertexRecoveryData(getVertex(vertexName).getVertexId());
                        Preconditions.checkArgument(vertexRecoveryData != null, "Vertex Group has been committed" + ", but no VertexRecoveryData found for its vertex " + vertexName);
                        VertexFinishedEvent vertexFinishedEvent = vertexRecoveryData.getVertexFinishedEvent();
                        Preconditions.checkArgument(vertexFinishedEvent != null, "Vertex Group has been committed" + ", but no VertexFinishedEvent found in its vertex " + vertexName);
                        Preconditions.checkArgument(vertexFinishedEvent.getState() == VertexState.SUCCEEDED, "Vertex Group has been committed, but unexpected vertex state of its vertex " + vertexName + ", vertexstate=" + vertexFinishedEvent.getState());
                    }
                    continue;
                }
                groupInfo.commitStarted = true;
                final Vertex v = getVertex(groupInfo.groupMembers.iterator().next());
                try {
                    Collection<TezVertexID> vertexIds = getVertexIds(groupInfo.groupMembers);
                    appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(getID(), new VertexGroupCommitStartedEvent(dagId, groupInfo.groupName, vertexIds, clock.getTime())));
                } catch (IOException e) {
                    LOG.error("Failed to send commit recovery event to handler", e);
                    recoveryFailed = true;
                }
                if (!recoveryFailed) {
                    for (final String outputName : groupInfo.outputs) {
                        OutputKey outputKey = new OutputKey(outputName, groupInfo.groupName, true);
                        CommitCallback groupCommitCallback = new CommitCallback(outputKey);
                        CallableEvent groupCommitCallableEvent = new CallableEvent(groupCommitCallback) {

                            public Void call() throws Exception {
                                OutputCommitter committer = v.getOutputCommitters().get(outputName);
                                LOG.info("Committing output: " + outputName);
                                commitOutput(committer);
                                return null;
                            }
                        };
                        ListenableFuture<Void> groupCommitFuture = appContext.getExecService().submit(groupCommitCallableEvent);
                        Futures.addCallback(groupCommitFuture, groupCommitCallableEvent.getCallback());
                        commitFutures.put(outputKey, groupCommitFuture);
                    }
                }
            }
        }
    }
    if (recoveryFailed) {
        LOG.info("Recovery failure occurred during commit");
        enactKill(DAGTerminationCause.RECOVERY_FAILURE, VertexTerminationCause.COMMIT_FAILURE);
    }
    return !recoveryFailed;
}

Example 3 with CallableEvent

use of org.apache.tez.dag.app.dag.event.CallableEvent in project tez by apache.

the class TestVertexImpl method setupPostDagCreation.

@SuppressWarnings({ "unchecked", "rawtypes" })
public void setupPostDagCreation() throws TezException {
    String dagName = "dag0";
    // dispatcher may be created multiple times (setupPostDagCreation may be called multiples)
    if (dispatcher != null) {
        dispatcher.stop();
    }
    dispatcher = new DrainDispatcher();
    appContext = mock(AppContext.class);
    when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim());
    when(appContext.getContainerLauncherName(anyInt())).thenReturn(TezConstants.getTezYarnServicePluginName());
    thh = mock(TaskHeartbeatHandler.class);
    historyEventHandler = mock(HistoryEventHandler.class);
    TaskSchedulerManager taskScheduler = mock(TaskSchedulerManager.class);
    UserGroupInformation ugi;
    try {
        ugi = UserGroupInformation.getCurrentUser();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    DAG dag = mock(DAG.class);
    doReturn(ugi).when(dag).getDagUGI();
    doReturn(dagName).when(dag).getName();
    Map<String, LocalResource> localResources = new HashMap<>();
    for (PlanLocalResource planLR : dagPlan.getLocalResourceList()) {
        localResources.put(planLR.getName(), DagTypeConverters.convertPlanLocalResourceToLocalResource(planLR));
    }
    when(dag.getLocalResources()).thenReturn(localResources);
    doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
    doReturn(appAttemptId.getApplicationId()).when(appContext).getApplicationID();
    doReturn(dag).when(appContext).getCurrentDAG();
    execService = mock(ListeningExecutorService.class);
    final ListenableFuture<Void> mockFuture = mock(ListenableFuture.class);
    Mockito.doAnswer(new Answer() {

        public ListenableFuture<Void> answer(InvocationOnMock invocation) {
            Object[] args = invocation.getArguments();
            CallableEvent e = (CallableEvent) args[0];
            dispatcher.getEventHandler().handle(e);
            return mockFuture;
        }
    }).when(execService).submit((Callable<Void>) any());
    MockClock clock = new MockClock();
    doReturn(execService).when(appContext).getExecService();
    doReturn(conf).when(appContext).getAMConf();
    doReturn(new Credentials()).when(dag).getCredentials();
    doReturn(DAGPlan.getDefaultInstance()).when(dag).getJobPlan();
    doReturn(dagId).when(appContext).getCurrentDAGID();
    doReturn(dagId).when(dag).getID();
    doReturn(taskScheduler).when(appContext).getTaskScheduler();
    doReturn(Resource.newInstance(102400, 60)).when(taskScheduler).getTotalResources(0);
    doReturn(historyEventHandler).when(appContext).getHistoryHandler();
    doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler();
    doReturn(clock).when(appContext).getClock();
    vertexGroups = Maps.newHashMap();
    for (PlanVertexGroupInfo groupInfo : dagPlan.getVertexGroupsList()) {
        vertexGroups.put(groupInfo.getGroupName(), new VertexGroupInfo(groupInfo));
    }
    // updateTracker may be created multiple times (setupPostDagCreation may be called multiples)
    if (updateTracker != null) {
        updateTracker.stop();
    }
    updateTracker = new StateChangeNotifierForTest(appContext.getCurrentDAG());
    setupVertices();
    when(dag.getVertex(any(TezVertexID.class))).thenAnswer(new Answer<Vertex>() {

        @Override
        public Vertex answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            if (args.length != 1) {
                return null;
            }
            TezVertexID vId = (TezVertexID) args[0];
            return vertexIdMap.get(vId);
        }
    });
    when(dag.getVertex(any(String.class))).thenAnswer(new Answer<Vertex>() {

        @Override
        public Vertex answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            if (args.length != 1) {
                return null;
            }
            String vId = (String) args[0];
            return vertices.get(vId);
        }
    });
    // TODO - this test logic is tightly linked to impl DAGImpl code.
    edges = new HashMap<String, Edge>();
    for (EdgePlan edgePlan : dagPlan.getEdgeList()) {
        EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(edgePlan);
        edges.put(edgePlan.getId(), new Edge(edgeProperty, dispatcher.getEventHandler(), conf));
    }
    parseVertexEdges();
    for (Edge edge : edges.values()) {
        edge.initialize();
    }
    dispatcher.register(CallableEventType.class, new CallableEventDispatcher());
    taskAttemptEventDispatcher = new TaskAttemptEventDispatcher();
    dispatcher.register(TaskAttemptEventType.class, taskAttemptEventDispatcher);
    taskEventDispatcher = new TaskEventDispatcher();
    dispatcher.register(TaskEventType.class, taskEventDispatcher);
    vertexEventDispatcher = new VertexEventDispatcher();
    dispatcher.register(VertexEventType.class, vertexEventDispatcher);
    dagEventDispatcher = new DagEventDispatcher();
    dispatcher.register(DAGEventType.class, dagEventDispatcher);
    amSchedulerEventDispatcher = new AMSchedulerEventDispatcher();
    dispatcher.register(AMSchedulerEventType.class, amSchedulerEventDispatcher);
    dispatcher.init(conf);
    dispatcher.start();
}

Example 4 with CallableEvent

use of org.apache.tez.dag.app.dag.event.CallableEvent in project tez by apache.

the class TestVertexManager method setup.

@Before
public void setup() {
    mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    execService = mock(ListeningExecutorService.class);
    final ListenableFuture<Void> mockFuture = mock(ListenableFuture.class);
    Mockito.doAnswer(new Answer() {

        public ListenableFuture<Void> answer(InvocationOnMock invocation) {
            Object[] args = invocation.getArguments();
            CallableEvent e = (CallableEvent) args[0];
            new CallableEventDispatcher().handle(e);
            return mockFuture;
        }
    }).when(execService).submit((Callable<Void>) any());
    doReturn(execService).when(mockAppContext).getExecService();
    mockVertex = mock(Vertex.class, RETURNS_DEEP_STUBS);
    doReturn("vertex1").when(mockVertex).getName();
    mockHandler = mock(EventHandler.class);
    when(mockAppContext.getEventHandler()).thenReturn(mockHandler);
    when(mockAppContext.getCurrentDAG().getVertex(any(String.class)).getTotalTasks()).thenReturn(1);
    requestCaptor = ArgumentCaptor.forClass(VertexEventInputDataInformation.class);
}

Example 5 with CallableEvent

use of org.apache.tez.dag.app.dag.event.CallableEvent in project tez by apache.

the class TestDAGRecovery method setup.

@SuppressWarnings({ "unchecked", "rawtypes" })
@Before
public void setup() {
    conf = new Configuration();
    conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false);
    appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(100, 1), 1);
    dagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 1);
    Assert.assertNotNull(dagId);
    dagPlan = createDAGPlan();
    dispatcher = new DrainDispatcher();
    fsTokens = new Credentials();
    appContext = mock(AppContext.class);
    execService = mock(ListeningExecutorService.class);
    thh = mock(TaskHeartbeatHandler.class);
    final ListenableFuture<Void> mockFuture = mock(ListenableFuture.class);
    when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim());
    when(appContext.getApplicationID()).thenReturn(appAttemptId.getApplicationId());
    when(appContext.getClock()).thenReturn(new SystemClock());
    Mockito.doAnswer(new Answer() {

        public ListenableFuture<Void> answer(InvocationOnMock invocation) {
            Object[] args = invocation.getArguments();
            CallableEvent e = (CallableEvent) args[0];
            dispatcher.getEventHandler().handle(e);
            return mockFuture;
        }
    }).when(execService).submit((Callable<Void>) any());
    doReturn(execService).when(appContext).getExecService();
    historyEventHandler = new MockHistoryEventHandler(appContext);
    aclManager = new ACLManager("amUser");
    doReturn(conf).when(appContext).getAMConf();
    doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
    doReturn(appAttemptId.getApplicationId()).when(appContext).getApplicationID();
    doReturn(dagId).when(appContext).getCurrentDAGID();
    doReturn(historyEventHandler).when(appContext).getHistoryHandler();
    doReturn(aclManager).when(appContext).getAMACLManager();
    doReturn(dagRecoveryData).when(appContext).getDAGRecoveryData();
    dag = new DAGImpl(dagId, conf, dagPlan, dispatcher.getEventHandler(), taskCommunicatorManagerInterface, fsTokens, clock, "user", thh, appContext);
    dag.entityUpdateTracker = new StateChangeNotifierForTest(dag);
    doReturn(dag).when(appContext).getCurrentDAG();
    ugi = mock(UserGroupInformation.class);
    UserGroupInformation ugi = dag.getDagUGI();
    doReturn(clusterInfo).when(appContext).getClusterInfo();
    TaskSchedulerManager mockTaskScheduler = mock(TaskSchedulerManager.class);
    doReturn(mockTaskScheduler).when(appContext).getTaskScheduler();
    v1Id = TezVertexID.getInstance(dagId, 0);
    t1v1Id = TezTaskID.getInstance(v1Id, 0);
    ta1t1v1Id = TezTaskAttemptID.getInstance(t1v1Id, 0);
    v2Id = TezVertexID.getInstance(dagId, 1);
    t1v2Id = TezTaskID.getInstance(v2Id, 0);
    ta1t1v2Id = TezTaskAttemptID.getInstance(t1v2Id, 0);
    dispatcher.register(CallableEventType.class, new CallableEventDispatcher());
    taskEventDispatcher = new TaskEventDispatcher();
    dispatcher.register(TaskEventType.class, taskEventDispatcher);
    taskAttemptEventDispatcher = new TaskAttemptEventDispatcher();
    dispatcher.register(TaskAttemptEventType.class, taskAttemptEventDispatcher);
    vertexEventDispatcher = new VertexEventDispatcher();
    dispatcher.register(VertexEventType.class, vertexEventDispatcher);
    dagEventDispatcher = new DagEventDispatcher();
    dispatcher.register(DAGEventType.class, dagEventDispatcher);
    dagFinishEventHandler = new DAGFinishEventHandler();
    dispatcher.register(DAGAppMasterEventType.class, dagFinishEventHandler);
    dispatcher.register(AMSchedulerEventType.class, new AMSchedulerEventDispatcher());
    dispatcher.init(conf);
    dispatcher.start();
    doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler();
    LogManager.getRootLogger().setLevel(Level.DEBUG);
}