Examples with DAGPlan org.apache.tez.dag.api.records.DAGProtos.DAGPlan used on opensource projects

Example 26 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class ATSHistoryLoggingService method getDomainForEvent.

private String getDomainForEvent(DAGHistoryEvent event) {
    String domainId = sessionDomainId;
    if (historyACLPolicyManager == null) {
        return domainId;
    }
    TezDAGID dagId = event.getDagID();
    HistoryEvent historyEvent = event.getHistoryEvent();
    if (dagId == null || !HistoryEventType.isDAGSpecificEvent(historyEvent.getEventType())) {
        return domainId;
    }
    if (dagDomainIdMap.containsKey(dagId)) {
        // If we already have the domain for the dag id return it
        domainId = dagDomainIdMap.get(dagId);
        // Cleanup if this is the last event.
        if (historyEvent.getEventType() == HistoryEventType.DAG_FINISHED) {
            dagDomainIdMap.remove(dagId);
        }
    } else if (HistoryEventType.DAG_SUBMITTED == historyEvent.getEventType() || HistoryEventType.DAG_RECOVERED == historyEvent.getEventType()) {
        // In case this is the first event for the dag, create and populate dag domain.
        Configuration conf;
        DAGPlan dagPlan;
        if (HistoryEventType.DAG_SUBMITTED == historyEvent.getEventType()) {
            conf = ((DAGSubmittedEvent) historyEvent).getConf();
            dagPlan = ((DAGSubmittedEvent) historyEvent).getDAGPlan();
        } else {
            conf = appContext.getCurrentDAG().getConf();
            dagPlan = appContext.getCurrentDAG().getJobPlan();
        }
        domainId = createDagDomain(conf, dagPlan, dagId);
        // createDagDomain updates skippedDAGs so another check here.
        if (skippedDAGs.contains(dagId)) {
            return null;
        }
        dagDomainIdMap.put(dagId, domainId);
    }
    return domainId;
}

Example 27 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestATSHistoryWithACLs method testDagLoggingDisabled.

/**
 * use mini cluster to verify data do not push to ats when the daglogging flag
 * in dagsubmittedevent is set off
 * @throws Exception
 */
@Test(timeout = 50000)
public void testDagLoggingDisabled() throws Exception {
    ATSHistoryLoggingService historyLoggingService;
    historyLoggingService = ReflectionUtils.createClazzInstance(ATSHistoryLoggingService.class.getName());
    AppContext appContext = mock(AppContext.class);
    when(appContext.getApplicationID()).thenReturn(ApplicationId.newInstance(0, 1));
    historyLoggingService.setAppContext(appContext);
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    String viewAcls = "nobody nobody_group";
    tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
    historyLoggingService.init(tezConf);
    historyLoggingService.start();
    ApplicationId appId = ApplicationId.newInstance(100l, 1);
    TezDAGID tezDAGID = TezDAGID.getInstance(appId, 100);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    DAGPlan dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build();
    DAGSubmittedEvent submittedEvent = new DAGSubmittedEvent(tezDAGID, 1, dagPlan, appAttemptId, null, "usr", tezConf, null, null);
    submittedEvent.setHistoryLoggingEnabled(false);
    DAGHistoryEvent event = new DAGHistoryEvent(tezDAGID, submittedEvent);
    historyLoggingService.handle(new DAGHistoryEvent(tezDAGID, submittedEvent));
    Thread.sleep(1000l);
    String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/" + event.getDagID();
    Client client = new Client();
    WebResource resource = client.resource(url);
    ClientResponse response = resource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(404, response.getStatus());
}

Example 28 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class ATSV15HistoryLoggingService method getDomainForEvent.

private String getDomainForEvent(DAGHistoryEvent event) {
    String domainId = sessionDomainId;
    if (historyACLPolicyManager == null) {
        return domainId;
    }
    TezDAGID dagId = event.getDagID();
    HistoryEvent historyEvent = event.getHistoryEvent();
    if (dagId == null || !HistoryEventType.isDAGSpecificEvent(historyEvent.getEventType())) {
        return domainId;
    }
    if (dagDomainIdMap.containsKey(dagId)) {
        // If we already have the domain for the dag id return it
        domainId = dagDomainIdMap.get(dagId);
        // Cleanup if this is the last event.
        if (historyEvent.getEventType() == HistoryEventType.DAG_FINISHED) {
            dagDomainIdMap.remove(dagId);
        }
    } else if (HistoryEventType.DAG_SUBMITTED == historyEvent.getEventType() || HistoryEventType.DAG_RECOVERED == historyEvent.getEventType()) {
        // In case this is the first event for the dag, create and populate dag domain.
        Configuration conf;
        DAGPlan dagPlan;
        if (HistoryEventType.DAG_SUBMITTED == historyEvent.getEventType()) {
            conf = ((DAGSubmittedEvent) historyEvent).getConf();
            dagPlan = ((DAGSubmittedEvent) historyEvent).getDAGPlan();
        } else {
            conf = appContext.getCurrentDAG().getConf();
            dagPlan = appContext.getCurrentDAG().getJobPlan();
        }
        domainId = createDagDomain(conf, dagPlan, dagId);
        // createDagDomain updates skippedDAGs so another check here.
        if (skippedDAGs.contains(dagId)) {
            return null;
        }
        dagDomainIdMap.put(dagId, domainId);
    }
    return domainId;
}

Example 29 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAGClientHandler method testDAGClientHandler.

@Test(timeout = 5000)
public void testDAGClientHandler() throws TezException {
    TezDAGID mockTezDAGId = mock(TezDAGID.class);
    when(mockTezDAGId.getId()).thenReturn(1);
    when(mockTezDAGId.toString()).thenReturn("dag_9999_0001_1");
    DAG mockDAG = mock(DAG.class);
    when(mockDAG.getID()).thenReturn(mockTezDAGId);
    DAGStatusBuilder mockDagStatusBuilder = mock(DAGStatusBuilder.class);
    when(mockDAG.getDAGStatus(anySetOf(StatusGetOpts.class))).thenReturn(mockDagStatusBuilder);
    VertexStatusBuilder mockVertexStatusBuilder = mock(VertexStatusBuilder.class);
    when(mockDAG.getVertexStatus(anyString(), anySetOf(StatusGetOpts.class))).thenReturn(mockVertexStatusBuilder);
    DAGAppMaster mockDagAM = mock(DAGAppMaster.class);
    when(mockDagAM.getState()).thenReturn(DAGAppMasterState.RUNNING);
    AppContext mockAppContext = mock(AppContext.class);
    when(mockDagAM.getContext()).thenReturn(mockAppContext);
    when(mockDagAM.getContext().getCurrentDAG()).thenReturn(mockDAG);
    when(mockAppContext.getClock()).thenReturn(new SystemClock());
    DAGClientHandler dagClientHandler = new DAGClientHandler(mockDagAM);
    // getAllDAGs()
    assertEquals(1, dagClientHandler.getAllDAGs().size());
    assertEquals("dag_9999_0001_1", dagClientHandler.getAllDAGs().get(0));
    // getDAGStatus
    try {
        dagClientHandler.getDAGStatus("dag_9999_0001_2", Sets.newSet(StatusGetOpts.GET_COUNTERS));
        fail("should not come here");
    } catch (TezException e) {
        assertTrue(e.getMessage().contains("Unknown dagId"));
    }
    DAGStatus dagStatus = dagClientHandler.getDAGStatus("dag_9999_0001_1", Sets.newSet(StatusGetOpts.GET_COUNTERS));
    assertEquals(mockDagStatusBuilder, dagStatus);
    // getVertexStatus
    try {
        dagClientHandler.getVertexStatus("dag_9999_0001_2", "v1", Sets.newSet(StatusGetOpts.GET_COUNTERS));
        fail("should not come here");
    } catch (TezException e) {
        assertTrue(e.getMessage().contains("Unknown dagId"));
    }
    VertexStatus vertexStatus = dagClientHandler.getVertexStatus("dag_9999_0001_1", "v1", Sets.newSet(StatusGetOpts.GET_COUNTERS));
    assertEquals(mockVertexStatusBuilder, vertexStatus);
    // getTezAppMasterStatus
    when(mockDagAM.isSession()).thenReturn(false);
    assertEquals(TezAppMasterStatus.RUNNING, dagClientHandler.getTezAppMasterStatus());
    when(mockDagAM.isSession()).thenReturn(true);
    when(mockDagAM.getState()).thenReturn(DAGAppMasterState.INITED);
    assertEquals(TezAppMasterStatus.INITIALIZING, dagClientHandler.getTezAppMasterStatus());
    when(mockDagAM.getState()).thenReturn(DAGAppMasterState.ERROR);
    assertEquals(TezAppMasterStatus.SHUTDOWN, dagClientHandler.getTezAppMasterStatus());
    // tryKillDAG
    try {
        dagClientHandler.tryKillDAG("dag_9999_0001_2");
        fail("should not come here");
    } catch (TezException e) {
        assertTrue(e.getMessage().contains("Unknown dagId"));
    }
    dagClientHandler.tryKillDAG("dag_9999_0001_1");
    ArgumentCaptor<DAG> eventCaptor = ArgumentCaptor.forClass(DAG.class);
    verify(mockDagAM, times(1)).tryKillDAG(eventCaptor.capture(), contains("Sending client kill from"));
    assertEquals(1, eventCaptor.getAllValues().size());
    assertTrue(eventCaptor.getAllValues().get(0) instanceof DAG);
    assertEquals("dag_9999_0001_1", ((DAG) eventCaptor.getAllValues().get(0)).getID().toString());
    // submitDAG
    DAGPlan dagPlan = DAGPlan.getDefaultInstance();
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    dagClientHandler.submitDAG(dagPlan, localResources);
    verify(mockDagAM).submitDAGToAppMaster(dagPlan, localResources);
    // shutdown
    dagClientHandler.shutdownAM();
    verify(mockDagAM).shutdownTezAM(contains("Received message to shutdown AM from"));
}

Example 30 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAGClientAMProtocolBlockingPBServerImpl method testSubmitDagInSessionWithLargeDagPlan.

@Test(timeout = 100000)
@SuppressWarnings("unchecked")
public void testSubmitDagInSessionWithLargeDagPlan() throws Exception {
    int maxIPCMsgSize = 1024;
    String dagPlanName = "dagplan-name";
    File requestFile = tmpFolder.newFile("request-file");
    TezConfiguration conf = new TezConfiguration();
    conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, maxIPCMsgSize);
    // Check with 70 MB (64 MB is CodedInputStream's default limit in earlier versions of protobuf)
    byte[] randomBytes = new byte[70 << 20];
    (new Random()).nextBytes(randomBytes);
    UserPayload payload = UserPayload.create(ByteBuffer.wrap(randomBytes));
    Vertex vertex = Vertex.create("V", ProcessorDescriptor.create("P").setUserPayload(payload), 1);
    DAGPlan dagPlan = DAG.create(dagPlanName).addVertex(vertex).createDag(conf, null, null, null, false);
    String lrName = "localResource";
    String scheme = "file";
    String host = "localhost";
    int port = 80;
    String path = "/test";
    URL lrURL = URL.newInstance(scheme, host, port, path);
    LocalResource localResource = LocalResource.newInstance(lrURL, LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1);
    Map<String, LocalResource> localResources = new HashMap<>();
    localResources.put(lrName, localResource);
    SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder().setDAGPlan(dagPlan).setAdditionalAmResources(DagTypeConverters.convertFromLocalResources(localResources));
    try (FileOutputStream fileOutputStream = new FileOutputStream(requestFile)) {
        requestBuilder.build().writeTo(fileOutputStream);
    }
    DAGClientHandler dagClientHandler = mock(DAGClientHandler.class);
    ACLManager aclManager = mock(ACLManager.class);
    DAGClientAMProtocolBlockingPBServerImpl serverImpl = spy(new DAGClientAMProtocolBlockingPBServerImpl(dagClientHandler, FileSystem.get(conf)));
    when(dagClientHandler.getACLManager()).thenReturn(aclManager);
    when(dagClientHandler.submitDAG((DAGPlan) any(), (Map<String, LocalResource>) any())).thenReturn("dag-id");
    when(aclManager.checkAMModifyAccess((UserGroupInformation) any())).thenReturn(true);
    requestBuilder.clear().setSerializedRequestPath(requestFile.getAbsolutePath());
    serverImpl.submitDAG(null, requestBuilder.build());
    ArgumentCaptor<DAGPlan> dagPlanCaptor = ArgumentCaptor.forClass(DAGPlan.class);
    verify(dagClientHandler).submitDAG(dagPlanCaptor.capture(), localResourcesCaptor.capture());
    dagPlan = dagPlanCaptor.getValue();
    localResources = localResourcesCaptor.getValue();
    assertEquals(dagPlan.getName(), dagPlanName);
    assertEquals(dagPlan.getVertexCount(), 1);
    assertTrue(dagPlan.getSerializedSize() > maxIPCMsgSize);
    assertArrayEquals(randomBytes, dagPlan.getVertex(0).getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray());
    assertEquals(localResources.size(), 1);
    assertTrue(localResources.containsKey(lrName));
    localResource = localResources.get(lrName);
    assertEquals(localResource.getType(), LocalResourceType.FILE);
    assertEquals(localResource.getVisibility(), LocalResourceVisibility.PUBLIC);
    lrURL = localResource.getResource();
    assertEquals(lrURL.getScheme(), scheme);
    assertEquals(lrURL.getHost(), host);
    assertEquals(lrURL.getPort(), port);
    assertEquals(lrURL.getFile(), path);
}