Examples with CallableEvent org.apache.tez.dag.app.dag.event.CallableEvent used on opensource projects

Example 6 with CallableEvent

use of org.apache.tez.dag.app.dag.event.CallableEvent in project tez by apache.

the class TestDAGImpl method setup.

@SuppressWarnings({ "unchecked", "rawtypes" })
@Before
public void setup() {
    conf = new Configuration();
    conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false);
    appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(100, 1), 1);
    dagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 1);
    Assert.assertNotNull(dagId);
    dagPlan = createTestDAGPlan();
    dispatcher = new DrainDispatcher();
    fsTokens = new Credentials();
    appContext = mock(AppContext.class);
    execService = mock(ListeningExecutorService.class);
    final ListenableFuture<Void> mockFuture = mock(ListenableFuture.class);
    when(appContext.getHadoopShim()).thenReturn(defaultShim);
    when(appContext.getApplicationID()).thenReturn(appAttemptId.getApplicationId());
    Mockito.doAnswer(new Answer() {

        public ListenableFuture<Void> answer(InvocationOnMock invocation) {
            Object[] args = invocation.getArguments();
            CallableEvent e = (CallableEvent) args[0];
            dispatcher.getEventHandler().handle(e);
            return mockFuture;
        }
    }).when(execService).submit((Callable<Void>) any());
    doReturn(execService).when(appContext).getExecService();
    historyEventHandler = mock(HistoryEventHandler.class);
    aclManager = new ACLManager("amUser");
    doReturn(conf).when(appContext).getAMConf();
    doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
    doReturn(appAttemptId.getApplicationId()).when(appContext).getApplicationID();
    doReturn(dagId).when(appContext).getCurrentDAGID();
    doReturn(historyEventHandler).when(appContext).getHistoryHandler();
    doReturn(aclManager).when(appContext).getAMACLManager();
    doReturn(defaultShim).when(appContext).getHadoopShim();
    dag = new DAGImpl(dagId, conf, dagPlan, dispatcher.getEventHandler(), taskCommunicatorManagerInterface, fsTokens, clock, "user", thh, appContext);
    dag.entityUpdateTracker = new StateChangeNotifierForTest(dag);
    doReturn(dag).when(appContext).getCurrentDAG();
    doReturn(clusterInfo).when(appContext).getClusterInfo();
    mrrAppContext = mock(AppContext.class);
    doReturn(aclManager).when(mrrAppContext).getAMACLManager();
    doReturn(execService).when(mrrAppContext).getExecService();
    doReturn(defaultShim).when(mrrAppContext).getHadoopShim();
    mrrDagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 2);
    mrrDagPlan = createTestMRRDAGPlan();
    mrrDag = new DAGImpl(mrrDagId, conf, mrrDagPlan, dispatcher.getEventHandler(), taskCommunicatorManagerInterface, fsTokens, clock, "user", thh, mrrAppContext);
    mrrDag.entityUpdateTracker = new StateChangeNotifierForTest(mrrDag);
    doReturn(conf).when(mrrAppContext).getAMConf();
    doReturn(mrrDag).when(mrrAppContext).getCurrentDAG();
    doReturn(appAttemptId).when(mrrAppContext).getApplicationAttemptId();
    doReturn(appAttemptId.getApplicationId()).when(mrrAppContext).getApplicationID();
    doReturn(historyEventHandler).when(mrrAppContext).getHistoryHandler();
    doReturn(clusterInfo).when(mrrAppContext).getClusterInfo();
    groupAppContext = mock(AppContext.class);
    doReturn(aclManager).when(groupAppContext).getAMACLManager();
    doReturn(execService).when(groupAppContext).getExecService();
    doReturn(defaultShim).when(groupAppContext).getHadoopShim();
    groupDagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 3);
    groupDagPlan = createGroupDAGPlan();
    groupDag = new DAGImpl(groupDagId, conf, groupDagPlan, dispatcher.getEventHandler(), taskCommunicatorManagerInterface, fsTokens, clock, "user", thh, groupAppContext);
    groupDag.entityUpdateTracker = new StateChangeNotifierForTest(groupDag);
    doReturn(conf).when(groupAppContext).getAMConf();
    doReturn(groupDag).when(groupAppContext).getCurrentDAG();
    doReturn(appAttemptId).when(groupAppContext).getApplicationAttemptId();
    doReturn(appAttemptId.getApplicationId()).when(groupAppContext).getApplicationID();
    doReturn(historyEventHandler).when(groupAppContext).getHistoryHandler();
    doReturn(clusterInfo).when(groupAppContext).getClusterInfo();
    // reset totalCommitCounter to 0
    TotalCountingOutputCommitter.totalCommitCounter = 0;
    dispatcher.register(CallableEventType.class, new CallableEventDispatcher());
    taskEventDispatcher = new TaskEventDispatcher();
    dispatcher.register(TaskEventType.class, taskEventDispatcher);
    taskAttemptEventDispatcher = new TaskAttemptEventDispatcher();
    dispatcher.register(TaskAttemptEventType.class, taskAttemptEventDispatcher);
    vertexEventDispatcher = new VertexEventDispatcher();
    dispatcher.register(VertexEventType.class, vertexEventDispatcher);
    dagEventDispatcher = new DagEventDispatcher();
    dispatcher.register(DAGEventType.class, dagEventDispatcher);
    dagFinishEventHandler = new DAGFinishEventHandler();
    dispatcher.register(DAGAppMasterEventType.class, dagFinishEventHandler);
    dispatcher.init(conf);
    dispatcher.start();
}

Example 7 with CallableEvent

use of org.apache.tez.dag.app.dag.event.CallableEvent in project tez by apache.

the class DAGImpl method commitOrFinish.

// either commit when all vertices are completed or just finish if there's no committer
private synchronized DAGState commitOrFinish() {
    // commit all other outputs
    // we come here for successful dag completion and when outputs need to be
    // committed at the end for all or none visibility
    Map<OutputKey, CallableEvent> commitEvents = new HashMap<OutputKey, CallableEvent>();
    // commit all shared outputs
    for (final VertexGroupInfo groupInfo : vertexGroups.values()) {
        if (!groupInfo.outputs.isEmpty()) {
            groupInfo.commitStarted = true;
            final Vertex v = getVertex(groupInfo.groupMembers.iterator().next());
            try {
                TezUtilsInternal.setHadoopCallerContext(appContext.getHadoopShim(), v.getVertexId());
                for (final String outputName : groupInfo.outputs) {
                    final OutputKey outputKey = new OutputKey(outputName, groupInfo.groupName, true);
                    CommitCallback groupCommitCallback = new CommitCallback(outputKey);
                    CallableEvent groupCommitCallableEvent = new CallableEvent(groupCommitCallback) {

                        @Override
                        public Void call() throws Exception {
                            OutputCommitter committer = v.getOutputCommitters().get(outputName);
                            LOG.info("Committing output: " + outputKey);
                            commitOutput(committer);
                            return null;
                        }
                    };
                    commitEvents.put(outputKey, groupCommitCallableEvent);
                }
            } finally {
                appContext.getHadoopShim().clearHadoopCallerContext();
            }
        }
    }
    for (final Vertex vertex : vertices.values()) {
        if (vertex.getOutputCommitters() == null) {
            LOG.info("No output committers for vertex: " + vertex.getLogIdentifier());
            continue;
        }
        Map<String, OutputCommitter> outputCommitters = new HashMap<String, OutputCommitter>(vertex.getOutputCommitters());
        Set<String> sharedOutputs = vertex.getSharedOutputs();
        // remove shared outputs
        if (sharedOutputs != null) {
            Iterator<Map.Entry<String, OutputCommitter>> iter = outputCommitters.entrySet().iterator();
            while (iter.hasNext()) {
                if (sharedOutputs.contains(iter.next().getKey())) {
                    iter.remove();
                }
            }
        }
        if (outputCommitters.isEmpty()) {
            LOG.info("No exclusive output committers for vertex: " + vertex.getLogIdentifier());
            continue;
        }
        try {
            TezUtilsInternal.setHadoopCallerContext(appContext.getHadoopShim(), vertex.getVertexId());
            for (final Map.Entry<String, OutputCommitter> entry : outputCommitters.entrySet()) {
                if (vertex.getState() != VertexState.SUCCEEDED) {
                    throw new TezUncheckedException("Vertex: " + vertex.getLogIdentifier() + " not in SUCCEEDED state. State= " + vertex.getState());
                }
                OutputKey outputKey = new OutputKey(entry.getKey(), vertex.getName(), false);
                CommitCallback commitCallback = new CommitCallback(outputKey);
                CallableEvent commitCallableEvent = new CallableEvent(commitCallback) {

                    @Override
                    public Void call() throws Exception {
                        LOG.info("Committing output: " + entry.getKey() + " for vertex: " + vertex.getLogIdentifier() + ", outputName: " + entry.getKey());
                        commitOutput(entry.getValue());
                        return null;
                    }
                };
                commitEvents.put(outputKey, commitCallableEvent);
            }
        } finally {
            appContext.getHadoopShim().clearHadoopCallerContext();
        }
    }
    if (!commitEvents.isEmpty()) {
        try {
            LOG.info("Start writing dag commit event, " + getID());
            appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(getID(), new DAGCommitStartedEvent(getID(), clock.getTime())));
        } catch (IOException e) {
            LOG.error("Failed to send commit event to history/recovery handler", e);
            trySetTerminationCause(DAGTerminationCause.RECOVERY_FAILURE);
            return finished(DAGState.FAILED);
        }
        for (Map.Entry<OutputKey, CallableEvent> entry : commitEvents.entrySet()) {
            ListenableFuture<Void> commitFuture = appContext.getExecService().submit(entry.getValue());
            Futures.addCallback(commitFuture, entry.getValue().getCallback());
            commitFutures.put(entry.getKey(), commitFuture);
        }
    }
    if (commitFutures.isEmpty()) {
        // no commit needs to be done
        return finished(DAGState.SUCCEEDED);
    } else {
        return DAGState.COMMITTING;
    }
}