Examples with TezDAGID org.apache.tez.dag.records.TezDAGID used on opensource projects

Example 46 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class ATSV15HistoryLoggingService method getDomainForEvent.

private String getDomainForEvent(DAGHistoryEvent event) {
    String domainId = sessionDomainId;
    if (historyACLPolicyManager == null) {
        return domainId;
    }
    TezDAGID dagId = event.getDagID();
    HistoryEvent historyEvent = event.getHistoryEvent();
    if (dagId == null || !HistoryEventType.isDAGSpecificEvent(historyEvent.getEventType())) {
        return domainId;
    }
    if (dagDomainIdMap.containsKey(dagId)) {
        // If we already have the domain for the dag id return it
        domainId = dagDomainIdMap.get(dagId);
        // Cleanup if this is the last event.
        if (historyEvent.getEventType() == HistoryEventType.DAG_FINISHED) {
            dagDomainIdMap.remove(dagId);
        }
    } else if (HistoryEventType.DAG_SUBMITTED == historyEvent.getEventType() || HistoryEventType.DAG_RECOVERED == historyEvent.getEventType()) {
        // In case this is the first event for the dag, create and populate dag domain.
        Configuration conf;
        DAGPlan dagPlan;
        if (HistoryEventType.DAG_SUBMITTED == historyEvent.getEventType()) {
            conf = ((DAGSubmittedEvent) historyEvent).getConf();
            dagPlan = ((DAGSubmittedEvent) historyEvent).getDAGPlan();
        } else {
            conf = appContext.getCurrentDAG().getConf();
            dagPlan = appContext.getCurrentDAG().getJobPlan();
        }
        domainId = createDagDomain(conf, dagPlan, dagId);
        // createDagDomain updates skippedDAGs so another check here.
        if (skippedDAGs.contains(dagId)) {
            return null;
        }
        dagDomainIdMap.put(dagId, domainId);
    }
    return domainId;
}

Example 47 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class ATSV15HistoryLoggingService method isValidEvent.

private boolean isValidEvent(DAGHistoryEvent event) {
    HistoryEventType eventType = event.getHistoryEvent().getEventType();
    TezDAGID dagId = event.getDagID();
    if (eventType.equals(HistoryEventType.DAG_SUBMITTED)) {
        DAGSubmittedEvent dagSubmittedEvent = (DAGSubmittedEvent) event.getHistoryEvent();
        String dagName = dagSubmittedEvent.getDAGName();
        if ((dagName != null && dagName.startsWith(TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) || (!dagSubmittedEvent.isHistoryLoggingEnabled())) {
            // Skip recording pre-warm DAG events
            skippedDAGs.add(dagId);
            return false;
        }
    }
    if (eventType.equals(HistoryEventType.DAG_RECOVERED)) {
        DAGRecoveredEvent dagRecoveredEvent = (DAGRecoveredEvent) event.getHistoryEvent();
        if (!dagRecoveredEvent.isHistoryLoggingEnabled()) {
            skippedDAGs.add(dagRecoveredEvent.getDagID());
            return false;
        }
    }
    if (eventType.equals(HistoryEventType.DAG_FINISHED)) {
        // No more events should be seen after this point.
        if (skippedDAGs.remove(dagId)) {
            return false;
        }
    }
    if (dagId != null && skippedDAGs.contains(dagId)) {
        // Skip pre-warm DAGs
        return false;
    }
    return true;
}

Example 48 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestATSV15HistoryLoggingService method testNonSessionDomainsFailed.

@Test
public void testNonSessionDomainsFailed() throws Exception {
    ATSV15HistoryLoggingService service = createService(-1);
    HistoryACLPolicyManager historyACLPolicyManager = mock(HistoryACLPolicyManager.class);
    service.historyACLPolicyManager = historyACLPolicyManager;
    when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), eq(appId))).thenThrow(new IOException());
    service.start();
    verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), eq(appId));
    // Send the event and wait for completion.
    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, service)) {
        service.handle(event);
    }
    while (!service.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }
    // No dag domain were created.
    verify(historyACLPolicyManager, times(0)).setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any());
    // History logging is disabled.
    verify(historyACLPolicyManager, times(0)).updateTimelineEntityDomain(any(), (String) any());
    assertEquals(0, entityLog.size());
    service.stop();
}

Example 49 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestATSV15HistoryLoggingService method testSessionDomainsFailed.

@Test
public void testSessionDomainsFailed() throws Exception {
    ATSV15HistoryLoggingService service = createService(-1);
    when(appContext.isSession()).thenReturn(true);
    HistoryACLPolicyManager historyACLPolicyManager = mock(HistoryACLPolicyManager.class);
    service.historyACLPolicyManager = historyACLPolicyManager;
    when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), eq(appId))).thenThrow(new IOException());
    service.start();
    // Verify that the session domain was created.
    verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), eq(appId));
    // Mock dag domain creation.
    when(historyACLPolicyManager.setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any())).thenReturn(Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID, "dag-id"));
    // Send the event and wait for completion.
    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, service)) {
        service.handle(event);
    }
    while (!service.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }
    // No dag creation was done.
    verify(historyACLPolicyManager, times(0)).setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any());
    // No history logging calls were done
    verify(historyACLPolicyManager, times(0)).updateTimelineEntityDomain(any(), (String) any());
    assertEquals(0, entityLog.size());
    service.stop();
}

Example 50 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestATSV15HistoryLoggingService method testDAGGroupingDisabled.

@Test(timeout = 2000)
public void testDAGGroupingDisabled() throws Exception {
    ATSV15HistoryLoggingService service = createService(1);
    service.start();
    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, service)) {
        service.handle(event);
    }
    while (!service.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }
    assertEquals(2, entityLog.size());
    List<TimelineEntity> amEvents = entityLog.get(TimelineEntityGroupId.newInstance(appId, appId.toString()));
    assertNotNull(amEvents);
    assertEquals(1, amEvents.size());
    List<TimelineEntity> nonGroupedDagEvents = entityLog.get(TimelineEntityGroupId.newInstance(appId, dagId1.toString()));
    assertNotNull(nonGroupedDagEvents);
    assertEquals(5, nonGroupedDagEvents.size());
    service.stop();
}