use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.
the class TestDAGUtils method testConvertDAGPlanToATSMap.
@Test(timeout = 5000)
@SuppressWarnings("unchecked")
public void testConvertDAGPlanToATSMap() throws IOException, JSONException {
DAGPlan dagPlan = createDAG();
Map<String, TezVertexID> idNameMap = new HashMap<String, TezVertexID>();
ApplicationId appId = ApplicationId.newInstance(1, 1);
TezDAGID dagId = TezDAGID.getInstance(appId, 1);
TezVertexID vId1 = TezVertexID.getInstance(dagId, 1);
TezVertexID vId2 = TezVertexID.getInstance(dagId, 2);
TezVertexID vId3 = TezVertexID.getInstance(dagId, 3);
idNameMap.put("vertex1", vId1);
idNameMap.put("vertex2", vId2);
idNameMap.put("vertex3", vId3);
Map<String, Object> atsMap = DAGUtils.convertDAGPlanToATSMap(dagPlan);
Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_NAME_KEY));
Assert.assertEquals("testDag", atsMap.get(DAGUtils.DAG_NAME_KEY));
Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_INFO_KEY));
Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_CONTEXT_KEY));
Map<String, String> contextMap = (Map<String, String>) atsMap.get(DAGUtils.DAG_CONTEXT_KEY);
Assert.assertEquals("context1", contextMap.get(ATSConstants.CONTEXT));
Assert.assertEquals("callerId1", contextMap.get(ATSConstants.CALLER_ID));
Assert.assertEquals("callerType1", contextMap.get(ATSConstants.CALLER_TYPE));
Assert.assertEquals("desc1", contextMap.get(ATSConstants.DESCRIPTION));
Assert.assertEquals("dagInfo", atsMap.get(DAGUtils.DAG_INFO_KEY));
Assert.assertEquals(dagPlan.getName(), atsMap.get(DAGUtils.DAG_NAME_KEY));
Assert.assertTrue(atsMap.containsKey("version"));
Assert.assertEquals(2, atsMap.get("version"));
Assert.assertTrue(atsMap.containsKey(DAGUtils.VERTICES_KEY));
Assert.assertTrue(atsMap.containsKey(DAGUtils.EDGES_KEY));
Assert.assertTrue(atsMap.containsKey(DAGUtils.VERTEX_GROUPS_KEY));
Assert.assertEquals(3, ((Collection<?>) atsMap.get(DAGUtils.VERTICES_KEY)).size());
Set<String> inEdgeIds = new HashSet<String>();
Set<String> outEdgeIds = new HashSet<String>();
int additionalInputCount = 0;
int additionalOutputCount = 0;
for (Object o : ((Collection<?>) atsMap.get(DAGUtils.VERTICES_KEY))) {
Map<String, Object> v = (Map<String, Object>) o;
Assert.assertTrue(v.containsKey(DAGUtils.VERTEX_NAME_KEY));
String vName = (String) v.get(DAGUtils.VERTEX_NAME_KEY);
Assert.assertTrue(v.containsKey(DAGUtils.PROCESSOR_CLASS_KEY));
Assert.assertTrue(v.containsKey(DAGUtils.USER_PAYLOAD_AS_TEXT));
if (v.containsKey(DAGUtils.IN_EDGE_IDS_KEY)) {
inEdgeIds.addAll(((Collection<String>) v.get(DAGUtils.IN_EDGE_IDS_KEY)));
}
if (v.containsKey(DAGUtils.OUT_EDGE_IDS_KEY)) {
outEdgeIds.addAll(((Collection<String>) v.get(DAGUtils.OUT_EDGE_IDS_KEY)));
}
Assert.assertTrue(idNameMap.containsKey(vName));
String procPayload = vName + " Processor HistoryText";
Assert.assertEquals(procPayload, v.get(DAGUtils.USER_PAYLOAD_AS_TEXT));
if (v.containsKey(DAGUtils.ADDITIONAL_INPUTS_KEY)) {
additionalInputCount += ((Collection<?>) v.get(DAGUtils.ADDITIONAL_INPUTS_KEY)).size();
for (Object input : ((Collection<?>) v.get(DAGUtils.ADDITIONAL_INPUTS_KEY))) {
Map<String, Object> inputMap = (Map<String, Object>) input;
Assert.assertTrue(inputMap.containsKey(DAGUtils.NAME_KEY));
Assert.assertTrue(inputMap.containsKey(DAGUtils.CLASS_KEY));
Assert.assertFalse(inputMap.containsKey(DAGUtils.INITIALIZER_KEY));
Assert.assertEquals("input HistoryText", inputMap.get(DAGUtils.USER_PAYLOAD_AS_TEXT));
}
}
if (v.containsKey(DAGUtils.ADDITIONAL_OUTPUTS_KEY)) {
additionalOutputCount += ((Collection<?>) v.get(DAGUtils.ADDITIONAL_OUTPUTS_KEY)).size();
for (Object output : ((Collection<?>) v.get(DAGUtils.ADDITIONAL_OUTPUTS_KEY))) {
Map<String, Object> outputMap = (Map<String, Object>) output;
Assert.assertTrue(outputMap.containsKey(DAGUtils.NAME_KEY));
Assert.assertTrue(outputMap.containsKey(DAGUtils.CLASS_KEY));
Assert.assertTrue(outputMap.containsKey(DAGUtils.INITIALIZER_KEY));
Assert.assertEquals("uvOut HistoryText", outputMap.get(DAGUtils.USER_PAYLOAD_AS_TEXT));
}
}
}
// 1 input
Assert.assertEquals(1, additionalInputCount);
// 3 outputs due to vertex group
Assert.assertEquals(3, additionalOutputCount);
// 1 edge translates to 2 due to vertex group
Assert.assertEquals(2, inEdgeIds.size());
Assert.assertEquals(2, outEdgeIds.size());
for (Object o : ((Collection<?>) atsMap.get(DAGUtils.EDGES_KEY))) {
Map<String, Object> e = (Map<String, Object>) o;
Assert.assertTrue(inEdgeIds.contains(e.get(DAGUtils.EDGE_ID_KEY)));
Assert.assertTrue(outEdgeIds.contains(e.get(DAGUtils.EDGE_ID_KEY)));
Assert.assertTrue(e.containsKey(DAGUtils.INPUT_VERTEX_NAME_KEY));
Assert.assertTrue(e.containsKey(DAGUtils.OUTPUT_VERTEX_NAME_KEY));
Assert.assertEquals(DataMovementType.SCATTER_GATHER.name(), e.get(DAGUtils.DATA_MOVEMENT_TYPE_KEY));
Assert.assertEquals(DataSourceType.PERSISTED.name(), e.get(DAGUtils.DATA_SOURCE_TYPE_KEY));
Assert.assertEquals(SchedulingType.SEQUENTIAL.name(), e.get(DAGUtils.SCHEDULING_TYPE_KEY));
Assert.assertEquals("dummy output class", e.get(DAGUtils.EDGE_SOURCE_CLASS_KEY));
Assert.assertEquals("dummy input class", e.get(DAGUtils.EDGE_DESTINATION_CLASS_KEY));
Assert.assertEquals("Dummy History Text", e.get(DAGUtils.OUTPUT_USER_PAYLOAD_AS_TEXT));
Assert.assertEquals("Dummy History Text", e.get(DAGUtils.INPUT_USER_PAYLOAD_AS_TEXT));
}
for (Object o : ((Collection<?>) atsMap.get(DAGUtils.VERTEX_GROUPS_KEY))) {
Map<String, Object> e = (Map<String, Object>) o;
Assert.assertEquals("uv12", e.get(DAGUtils.VERTEX_GROUP_NAME_KEY));
Assert.assertTrue(e.containsKey(DAGUtils.VERTEX_GROUP_MEMBERS_KEY));
Assert.assertTrue(e.containsKey(DAGUtils.VERTEX_GROUP_OUTPUTS_KEY));
Assert.assertTrue(e.containsKey(DAGUtils.VERTEX_GROUP_EDGE_MERGED_INPUTS_KEY));
}
}
use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.
the class TezClientUtils method createApplicationSubmissionContext.
/**
* Create an ApplicationSubmissionContext to launch a Tez AM
* @param appId Application Id
* @param dag DAG to be submitted
* @param amName Name for the application
* @param amConfig AM Configuration
* @param tezJarResources Resources to be used by the AM
* @param sessionCreds the credential object which will be populated with session specific
* @param servicePluginsDescriptor descriptor for services which may be running in the AM
* @return an ApplicationSubmissionContext to launch a Tez AM
* @throws IOException
* @throws YarnException
*/
@Private
@VisibleForTesting
public static ApplicationSubmissionContext createApplicationSubmissionContext(ApplicationId appId, DAG dag, String amName, AMConfiguration amConfig, Map<String, LocalResource> tezJarResources, Credentials sessionCreds, boolean tezLrsAsArchive, TezApiVersionInfo apiVersionInfo, ServicePluginsDescriptor servicePluginsDescriptor, JavaOptsChecker javaOptsChecker) throws IOException, YarnException {
Preconditions.checkNotNull(sessionCreds);
TezConfiguration conf = amConfig.getTezConfiguration();
FileSystem fs = TezClientUtils.ensureStagingDirExists(conf, TezCommonUtils.getTezBaseStagingPath(conf));
String strAppId = appId.toString();
Path tezSysStagingPath = TezCommonUtils.createTezSystemStagingPath(conf, strAppId);
Path binaryConfPath = TezCommonUtils.getTezConfStagingPath(tezSysStagingPath);
binaryConfPath = fs.makeQualified(binaryConfPath);
// Setup resource requirements
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB_DEFAULT));
capability.setVirtualCores(amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES_DEFAULT));
if (LOG.isDebugEnabled()) {
LOG.debug("AppMaster capability = " + capability);
}
// Setup required Credentials for the AM launch. DAG specific credentials
// are handled separately.
ByteBuffer securityTokens = null;
// Setup security tokens
Credentials amLaunchCredentials = new Credentials();
if (amConfig.getCredentials() != null) {
amLaunchCredentials.addAll(amConfig.getCredentials());
}
// Add Staging dir creds to the list of session credentials.
TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[] { binaryConfPath }, conf);
// Add session specific credentials to the AM credentials.
amLaunchCredentials.mergeAll(sessionCreds);
DataOutputBuffer dob = new DataOutputBuffer();
amLaunchCredentials.writeTokenStorageToStream(dob);
securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
// Setup the command to run the AM
List<String> vargs = new ArrayList<String>(8);
vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
String amOpts = constructAMLaunchOpts(amConfig.getTezConfiguration(), capability);
vargs.add(amOpts);
String amLogLevelString = amConfig.getTezConfiguration().get(TezConfiguration.TEZ_AM_LOG_LEVEL, TezConfiguration.TEZ_AM_LOG_LEVEL_DEFAULT);
String[] amLogParams = parseLogParams(amLogLevelString);
String amLogLevel = amLogParams[0];
maybeAddDefaultLoggingJavaOpts(amLogLevel, vargs);
// FIX sun bug mentioned in TEZ-327
vargs.add("-Dsun.nio.ch.bugLevel=''");
vargs.add(TezConstants.TEZ_APPLICATION_MASTER_CLASS);
if (dag == null) {
vargs.add("--" + TezConstants.TEZ_SESSION_MODE_CLI_OPTION);
}
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + ApplicationConstants.STDOUT);
vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + ApplicationConstants.STDERR);
Vector<String> vargsFinal = new Vector<String>(8);
// Final command
StringBuilder mergedCommand = new StringBuilder();
for (CharSequence str : vargs) {
mergedCommand.append(str).append(" ");
}
vargsFinal.add(mergedCommand.toString());
if (LOG.isDebugEnabled()) {
LOG.debug("Command to launch container for ApplicationMaster is : " + mergedCommand);
}
Map<String, String> environment = new TreeMap<String, String>();
TezYARNUtils.setupDefaultEnv(environment, conf, TezConfiguration.TEZ_AM_LAUNCH_ENV, TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT, TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV, TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV_DEFAULT, tezLrsAsArchive);
addVersionInfoToEnv(environment, apiVersionInfo);
addLogParamsToEnv(environment, amLogParams);
Map<String, LocalResource> amLocalResources = new TreeMap<String, LocalResource>();
// Not fetching credentials for AMLocalResources. Expect this to be provided via AMCredentials.
if (amConfig.getAMLocalResources() != null) {
amLocalResources.putAll(amConfig.getAMLocalResources());
}
amLocalResources.putAll(tezJarResources);
TezConfiguration tezConf = amConfig.getTezConfiguration();
// Merge the dag access controls into tez am config.
if (dag != null && dag.getDagAccessControls() != null) {
// Merge updates the conf object passed. In non session mode, same client object can be used
// to submit multiple dags, copying this prevents ACL of one DAG from being used in another.
tezConf = new TezConfiguration(amConfig.getTezConfiguration());
dag.getDagAccessControls().mergeIntoAmAcls(tezConf);
}
// don't overwrite existing conf, needed for TezClient.getClient() so existing containers have stable resource fingerprints
if (!binaryConfPath.getFileSystem(tezConf).exists(binaryConfPath)) {
ConfigurationProto finalConfProto = createFinalConfProtoForApp(tezConf, servicePluginsDescriptor);
FSDataOutputStream amConfPBOutBinaryStream = null;
try {
amConfPBOutBinaryStream = TezCommonUtils.createFileForAM(fs, binaryConfPath);
finalConfProto.writeTo(amConfPBOutBinaryStream);
} finally {
if (amConfPBOutBinaryStream != null) {
amConfPBOutBinaryStream.close();
}
}
}
LocalResource binaryConfLRsrc = TezClientUtils.createLocalResource(fs, binaryConfPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION);
amConfig.setBinaryConfLR(binaryConfLRsrc);
amLocalResources.put(TezConstants.TEZ_PB_BINARY_CONF_NAME, binaryConfLRsrc);
// Create Session Jars definition to be sent to AM as a local resource
Path sessionJarsPath = TezCommonUtils.getTezAMJarStagingPath(tezSysStagingPath);
FSDataOutputStream sessionJarsPBOutStream = null;
try {
sessionJarsPBOutStream = TezCommonUtils.createFileForAM(fs, sessionJarsPath);
// Write out the initial list of resources which will be available in the AM
DAGProtos.PlanLocalResourcesProto amResourceProto;
if (amLocalResources != null && !amLocalResources.isEmpty()) {
amResourceProto = DagTypeConverters.convertFromLocalResources(amLocalResources);
} else {
amResourceProto = DAGProtos.PlanLocalResourcesProto.getDefaultInstance();
}
amResourceProto.writeDelimitedTo(sessionJarsPBOutStream);
} finally {
if (sessionJarsPBOutStream != null) {
sessionJarsPBOutStream.close();
}
}
LocalResource sessionJarsPBLRsrc = TezClientUtils.createLocalResource(fs, sessionJarsPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION);
amLocalResources.put(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME, sessionJarsPBLRsrc);
String user = UserGroupInformation.getCurrentUser().getShortUserName();
ACLManager aclManager = new ACLManager(user, amConfig.getTezConfiguration());
Map<ApplicationAccessType, String> acls = aclManager.toYARNACls();
if (dag != null) {
DAGPlan dagPB = prepareAndCreateDAGPlan(dag, amConfig, tezJarResources, tezLrsAsArchive, sessionCreds, servicePluginsDescriptor, javaOptsChecker);
// emit protobuf DAG file style
Path binaryPath = TezCommonUtils.getTezBinPlanStagingPath(tezSysStagingPath);
if (LOG.isDebugEnabled()) {
LOG.debug("Stage directory information for AppId :" + appId + " tezSysStagingPath :" + tezSysStagingPath + " binaryConfPath :" + binaryConfPath + " sessionJarsPath :" + sessionJarsPath + " binaryPlanPath :" + binaryPath);
}
FSDataOutputStream dagPBOutBinaryStream = null;
try {
// binary output
dagPBOutBinaryStream = TezCommonUtils.createFileForAM(fs, binaryPath);
dagPB.writeTo(dagPBOutBinaryStream);
} finally {
if (dagPBOutBinaryStream != null) {
dagPBOutBinaryStream.close();
}
}
amLocalResources.put(TezConstants.TEZ_PB_PLAN_BINARY_NAME, TezClientUtils.createLocalResource(fs, binaryPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
if (Level.DEBUG.isGreaterOrEqual(Level.toLevel(amLogLevel))) {
Path textPath = localizeDagPlanAsText(dagPB, fs, amConfig, strAppId, tezSysStagingPath);
amLocalResources.put(TezConstants.TEZ_PB_PLAN_TEXT_NAME, TezClientUtils.createLocalResource(fs, textPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
}
}
// Send the shuffle token as part of the AM launch context, so that the NM running the AM can
// provide this to AuxServices running on the AM node - in case tasks run within the AM,
// and no other task runs on this node.
Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
String auxiliaryService = conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT);
serviceData.put(auxiliaryService, TezCommonUtils.serializeServiceData(TokenCache.getSessionToken(amLaunchCredentials)));
// Setup ContainerLaunchContext for AM container
ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(amLocalResources, environment, vargsFinal, serviceData, securityTokens, acls);
// Set up the ApplicationSubmissionContext
ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
Collection<String> tagsFromConf = amConfig.getTezConfiguration().getTrimmedStringCollection(TezConfiguration.TEZ_APPLICATION_TAGS);
appContext.setApplicationType(TezConstants.TEZ_APPLICATION_TYPE);
if (tagsFromConf != null && !tagsFromConf.isEmpty()) {
appContext.setApplicationTags(new HashSet<String>(tagsFromConf));
}
appContext.setApplicationId(appId);
appContext.setResource(capability);
String queueName = amConfig.getQueueName();
if (queueName != null && !queueName.isEmpty()) {
appContext.setQueue(amConfig.getQueueName());
}
// set the application priority
setApplicationPriority(appContext, amConfig);
appContext.setApplicationName(amName);
appContext.setCancelTokensWhenComplete(amConfig.getTezConfiguration().getBoolean(TezConfiguration.TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION, TezConfiguration.TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION_DEFAULT));
appContext.setAMContainerSpec(amContainer);
appContext.setMaxAppAttempts(amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT));
return appContext;
}
use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.
the class TezClient method submitDAGSession.
private DAGClient submitDAGSession(DAG dag) throws TezException, IOException {
Preconditions.checkState(isSession == true, "submitDAG with additional resources applies to only session mode. " + "In non-session mode please specify all resources in the initial configuration");
verifySessionStateForSubmission();
String dagId = null;
String callerContextStr = "";
if (dag.getCallerContext() != null) {
callerContextStr = ", callerContext=" + dag.getCallerContext().contextAsSimpleString();
}
LOG.info("Submitting dag to TezSession" + ", sessionName=" + clientName + ", applicationId=" + sessionAppId + ", dagName=" + dag.getName() + callerContextStr);
if (!additionalLocalResources.isEmpty()) {
for (LocalResource lr : additionalLocalResources.values()) {
Preconditions.checkArgument(lr.getType() == LocalResourceType.FILE, "LocalResourceType: " + lr.getType() + " is not supported, only " + LocalResourceType.FILE + " is supported");
}
}
Map<String, LocalResource> tezJarResources = getTezJarResources(sessionCredentials);
DAGPlan dagPlan = TezClientUtils.prepareAndCreateDAGPlan(dag, amConfig, tezJarResources, usingTezArchiveDeploy, sessionCredentials, servicePluginsDescriptor, javaOptsChecker);
SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder();
requestBuilder.setDAGPlan(dagPlan);
if (!additionalLocalResources.isEmpty()) {
requestBuilder.setAdditionalAmResources(DagTypeConverters.convertFromLocalResources(additionalLocalResources));
}
additionalLocalResources.clear();
// if request size exceeds maxSubmitDAGRequestSizeThroughIPC, we serialize them to HDFS
SubmitDAGRequestProto request = requestBuilder.build();
if (request.getSerializedSize() > maxSubmitDAGRequestSizeThroughIPC) {
Path dagPlanPath = new Path(TezCommonUtils.getTezSystemStagingPath(amConfig.getTezConfiguration(), sessionAppId.toString()), TezConstants.TEZ_PB_PLAN_BINARY_NAME + serializedSubmitDAGPlanRequestCounter.incrementAndGet());
try (FSDataOutputStream fsDataOutputStream = stagingFs.create(dagPlanPath, false)) {
LOG.info("Send dag plan using YARN local resources since it's too large" + ", dag plan size=" + request.getSerializedSize() + ", max dag plan size through IPC=" + maxSubmitDAGRequestSizeThroughIPC + ", max IPC message size= " + amConfig.getTezConfiguration().getInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT));
request.writeTo(fsDataOutputStream);
request = requestBuilder.clear().setSerializedRequestPath(stagingFs.resolvePath(dagPlanPath).toString()).build();
}
}
DAGClientAMProtocolBlockingPB proxy = null;
try {
proxy = waitForProxy();
} catch (InterruptedException e) {
throw new IOException("Interrupted while trying to create a connection to the AM", e);
}
if (proxy == null) {
try {
LOG.warn("DAG submission to session timed out, stopping session");
stop();
} catch (Throwable t) {
LOG.info("Got an exception when trying to stop session", t);
}
throw new DAGSubmissionTimedOut("Could not submit DAG to Tez Session" + ", timed out after " + clientTimeout + " seconds");
}
try {
SubmitDAGResponseProto response = proxy.submitDAG(null, request);
// SubmitDAGResponseProto cannot be mocked
if (response != null) {
dagId = response.getDagId();
}
} catch (ServiceException e) {
RPCUtil.unwrapAndThrowException(e);
}
LOG.info("Submitted dag to TezSession" + ", sessionName=" + clientName + ", applicationId=" + sessionAppId + ", dagId=" + dagId + ", dagName=" + dag.getName());
return new DAGClientImpl(sessionAppId, dagId, amConfig.getTezConfiguration(), amConfig.getYarnConfiguration(), frameworkClient);
}
use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.
the class TestATSHistoryWithACLs method testDagLoggingEnabled.
/**
* use mini cluster to verify data do push to ats when
* the dag logging flag in dagsubmitted event is set on
* @throws Exception
*/
@Test(timeout = 50000)
public void testDagLoggingEnabled() throws Exception {
ATSHistoryLoggingService historyLoggingService;
historyLoggingService = ReflectionUtils.createClazzInstance(ATSHistoryLoggingService.class.getName());
AppContext appContext = mock(AppContext.class);
when(appContext.getApplicationID()).thenReturn(ApplicationId.newInstance(0, 1));
historyLoggingService.setAppContext(appContext);
TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
String viewAcls = "nobody nobody_group";
tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
remoteFs.mkdirs(remoteStagingDir);
tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
historyLoggingService.init(tezConf);
historyLoggingService.start();
ApplicationId appId = ApplicationId.newInstance(100l, 1);
TezDAGID tezDAGID = TezDAGID.getInstance(appId, 11);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
DAGPlan dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build();
DAGSubmittedEvent submittedEvent = new DAGSubmittedEvent(tezDAGID, 1, dagPlan, appAttemptId, null, "usr", tezConf, null, null);
submittedEvent.setHistoryLoggingEnabled(true);
DAGHistoryEvent event = new DAGHistoryEvent(tezDAGID, submittedEvent);
historyLoggingService.handle(new DAGHistoryEvent(tezDAGID, submittedEvent));
Thread.sleep(1000l);
String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/" + event.getDagID();
Client client = new Client();
WebResource resource = client.resource(url);
ClientResponse response = resource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(200, response.getStatus());
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
TimelineEntity entity = response.getEntity(TimelineEntity.class);
assertEquals(entity.getEntityType(), "TEZ_DAG_ID");
assertEquals(entity.getEvents().get(0).getEventType(), HistoryEventType.DAG_SUBMITTED.toString());
}
use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.
the class DAGAppMaster method startDAG.
private void startDAG() throws IOException, TezException {
FileInputStream dagPBBinaryStream = null;
try {
DAGPlan dagPlan = null;
// Read the protobuf DAG
dagPBBinaryStream = new FileInputStream(new File(workingDirectory, TezConstants.TEZ_PB_PLAN_BINARY_NAME));
dagPlan = DAGPlan.parseFrom(dagPBBinaryStream);
startDAG(dagPlan, null);
} finally {
if (dagPBBinaryStream != null) {
dagPBBinaryStream.close();
}
}
}
Aggregations