use of datawave.webservice.mr.configuration.OozieJobConfiguration in project datawave by NationalSecurityAgency.
the class MapReduceBean method ooziesubmit.
/**
* Execute a Oozie workflow with the given workFlow name and runtime parameters
*
* @param queryParameters
* @return
*/
@POST
@Produces({ "application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff" })
@javax.ws.rs.Path("/ooziesubmit")
@GZIP
public GenericResponse<String> ooziesubmit(MultivaluedMap<String, String> queryParameters) {
GenericResponse<String> response = new GenericResponse<>();
String workFlow = queryParameters.getFirst(OozieJobConstants.WORKFLOW_PARAM);
if (StringUtils.isBlank(workFlow)) {
throw new BadRequestException(new IllegalArgumentException(OozieJobConstants.WORKFLOW_PARAM + " parameter missing"), response);
}
String parameters = queryParameters.getFirst(OozieJobConstants.PARAMETERS);
// Find out who/what called this method
Principal p = ctx.getCallerPrincipal();
String sid = null;
String userDn = p.getName();
DatawavePrincipal datawavePrincipal = null;
if (p instanceof DatawavePrincipal) {
datawavePrincipal = (DatawavePrincipal) p;
sid = datawavePrincipal.getShortName();
} else {
QueryException qe = new QueryException(DatawaveErrorCode.UNEXPECTED_PRINCIPAL_ERROR, MessageFormat.format("Class: {0}", p.getClass().getName()));
response.addException(qe);
throw new DatawaveWebApplicationException(qe, response);
}
OozieJobConfiguration job;
try {
MapReduceJobConfiguration mrConfig = this.mapReduceConfiguration.getConfiguration(workFlow);
if (mrConfig instanceof OozieJobConfiguration) {
job = (OozieJobConfiguration) mrConfig;
} else {
throw new IllegalArgumentException(workFlow + " not an Oozie job configuration");
}
} catch (IllegalArgumentException e) {
BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.JOB_CONFIGURATION_ERROR, e);
response.addException(qe);
throw new BadRequestException(qe, response);
}
if (job instanceof NeedCallerDetails) {
((NeedCallerDetails) job).setUserSid(sid);
((NeedCallerDetails) job).setPrincipal(p);
}
// Ensure that the user has the required roles and has passed the required auths
if (null != job.getRequiredRoles() || null != job.getRequiredAuths()) {
try {
canRunJob(datawavePrincipal, queryParameters, job.getRequiredRoles(), job.getRequiredAuths());
} catch (UnauthorizedQueryException qe) {
// user does not have all of the required roles or did not pass the required auths
response.addException(qe);
throw new UnauthorizedException(qe, response);
}
}
String id = sid + "_" + UUID.randomUUID();
OozieClient oozieClient = null;
Properties oozieConf = null;
try {
oozieClient = new OozieClient((String) job.getJobConfigurationProperties().get(OozieJobConstants.OOZIE_CLIENT_PROP));
oozieConf = oozieClient.createConfiguration();
job.initializeOozieConfiguration(id, oozieConf, queryParameters);
job.validateWorkflowParameter(oozieConf, mapReduceConfiguration);
} catch (QueryException qe) {
log.error(qe.getMessage(), qe);
response.addException(qe);
throw new DatawaveWebApplicationException(qe, response);
} catch (Exception e) {
log.error(e.getMessage(), e);
response.addException(new QueryException(e.getMessage(), e));
throw new DatawaveWebApplicationException(e, response);
} finally {
// audit query here
Auditor.AuditType auditType = job.getAuditType();
log.trace("Audit type is: " + auditType.name());
if (!auditType.equals(Auditor.AuditType.NONE)) {
try {
marking.validate(queryParameters);
PrivateAuditConstants.stripPrivateParameters(queryParameters);
queryParameters.putSingle(PrivateAuditConstants.USER_DN, userDn);
queryParameters.putSingle(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toColumnVisibilityString());
queryParameters.putSingle(PrivateAuditConstants.AUDIT_TYPE, auditType.name());
List<String> selectors = job.getSelectors(queryParameters, oozieConf);
if (selectors != null && !selectors.isEmpty()) {
queryParameters.put(PrivateAuditConstants.SELECTORS, selectors);
}
// if the user didn't set an audit id, use the query id
if (!queryParameters.containsKey(AuditParameters.AUDIT_ID)) {
queryParameters.putSingle(AuditParameters.AUDIT_ID, id);
}
auditor.audit(queryParameters);
} catch (IllegalArgumentException e) {
log.error("Error validating audit parameters", e);
BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER, e);
response.addException(qe);
throw new BadRequestException(qe, response);
} catch (Exception e) {
log.error("Error auditing query", e);
response.addMessage("Error auditing query - " + e.getMessage());
throw new BadRequestException(e, response);
}
}
}
// Submit the Oozie workflow.
try {
String jobID = null;
try {
jobID = oozieClient.run(oozieConf);
} catch (Exception e) {
throw new QueryException(DatawaveErrorCode.OOZIE_JOB_START_ERROR, e);
}
try {
String jobResultstDir = oozieConf.getProperty(OozieJobConstants.OUT_DIR_PROP) + "/" + id;
response.setResult(id);
Path baseDir = new Path(this.mapReduceConfiguration.getMapReduceBaseDirectory());
// Create a directory path for this job
Path jobDir = new Path(baseDir, id);
mapReduceState.create(id, job.getHdfsUri(), job.getJobTracker(), jobDir.toString(), jobID, jobResultstDir, parameters, workFlow);
} catch (Exception e) {
QueryException qe = new QueryException(DatawaveErrorCode.MAPREDUCE_STATE_PERSISTENCE_ERROR, e);
response.addException(qe.getBottomQueryException());
try {
oozieClient.kill(jobID);
// if we successfully kill the job, throw the original exception
throw qe;
} catch (Exception e2) {
// throw the exception from killing the job
throw new QueryException(DatawaveErrorCode.MAPREDUCE_JOB_KILL_ERROR, e2);
}
}
} catch (QueryException qe) {
log.error(qe.getMessage(), qe);
response.addException(qe);
throw new DatawaveWebApplicationException(qe, response);
} catch (Exception e) {
log.error(e.getMessage(), e);
QueryException qe = new QueryException(DatawaveErrorCode.UNKNOWN_SERVER_ERROR, e.getMessage());
response.addException(qe);
throw new DatawaveWebApplicationException(qe, response);
}
return response;
}
Aggregations