Examples with MapReduceInfoResponseList datawave.webservice.results.mr.MapReduceInfoResponseList used on opensource projects

Example 11 with MapReduceInfoResponseList

use of datawave.webservice.results.mr.MapReduceInfoResponseList in project datawave by NationalSecurityAgency.

the class MapReduceBean method remove.

/**
 * Removes the MapReduce entry and associated data
 *
 * @param jobId
 * @return datawave.webservice.result.VoidResponse
 * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user by specifying a chain of DNs of the identities to proxy
 * @RequestHeader X-ProxiedIssuersChain required when using X-ProxiedEntitiesChain, specify one issuer DN per subject DN listed in X-ProxiedEntitiesChain
 * @ResponseHeader X-OperationTimeInMS time spent on the server performing the operation, does not account for network or result serialization
 * @HTTP 200 success
 * @HTTP 404 if jobId is invalid or cannot be found
 * @HTTP 500 error removing the job
 */
@DELETE
@Produces({ "application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff" })
@javax.ws.rs.Path("/{jobId}/remove")
@GZIP
public VoidResponse remove(@PathParam("jobId") String jobId) {
    VoidResponse response = new VoidResponse();
    // Call cancel which will kill any running jobs and remove the results directory in HDFS.
    cancel(jobId);
    // Remove the working directory from HDFS
    MapReduceInfoResponseList list = list(jobId);
    MapReduceInfoResponse result = list.getResults().get(0);
    String hdfs = result.getHdfs();
    String wdir = result.getWorkingDirectory();
    Path p = new Path(wdir);
    try {
        FileSystem fs = getFS(hdfs, response);
        if (fs.exists(p) && !fs.delete(p, true)) {
            NotFoundQueryException qe = new NotFoundQueryException(DatawaveErrorCode.WORKING_DIRECTORY_DELETE_ERROR, MessageFormat.format("{0}", wdir));
            log.error(qe);
            response.addException(qe);
            throw new NotFoundException(qe, response);
        }
    } catch (IOException e) {
        NotFoundQueryException qe = new NotFoundQueryException(DatawaveErrorCode.WORKING_DIRECTORY_DELETE_ERROR, e, MessageFormat.format("{0}", wdir));
        log.error(qe);
        throw new NotFoundException(qe, response);
    }
    // Remove any persisted state information
    try {
        mapReduceState.remove(jobId);
    } catch (QueryException e) {
        log.error("Error removing job state information", e);
        response.addException(e.getBottomQueryException());
        throw new DatawaveWebApplicationException(e, response);
    }
    return response;
}

Example 12 with MapReduceInfoResponseList

use of datawave.webservice.results.mr.MapReduceInfoResponseList in project datawave by NationalSecurityAgency.

the class MapReduceStatePersisterTest method testFindNoResults.

@Test
public void testFindNoResults() throws Exception {
    EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal);
    HashMap<String, String> trackingMap = new HashMap<>();
    expect(connectionFactory.getTrackingMap(EasyMock.anyObject())).andReturn(trackingMap);
    expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.ADMIN), EasyMock.eq(trackingMap))).andReturn(connection);
    connectionFactory.returnConnection(connection);
    replayAll();
    MapReduceInfoResponseList result = bean.find();
    verifyAll();
    assertEquals(0, result.getResults().size());
}

Example 13 with MapReduceInfoResponseList

use of datawave.webservice.results.mr.MapReduceInfoResponseList in project datawave by NationalSecurityAgency.

the class MapReduceStatePersisterTest method testFindById.

@Test
public void testFindById() throws Exception {
    // create the initial entry
    testPersistentCreate();
    PowerMock.resetAll();
    EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal);
    HashMap<String, String> trackingMap = new HashMap<>();
    expect(connectionFactory.getTrackingMap(EasyMock.anyObject())).andReturn(trackingMap);
    expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.ADMIN), EasyMock.eq(trackingMap))).andReturn(connection);
    connectionFactory.returnConnection(connection);
    replayAll();
    MapReduceInfoResponseList result = bean.findById(id);
    verifyAll();
    assertEquals(1, result.getResults().size());
    assertNull(result.getExceptions());
    MapReduceInfoResponse response = result.getResults().get(0);
    assertEquals(id, response.getId());
    assertEquals(hdfs, response.getHdfs());
    assertEquals(jt, response.getJobTracker());
    assertEquals(jobName, response.getJobName());
    assertEquals(workingDirectory, response.getWorkingDirectory());
    assertEquals(resultsDirectory, response.getResultsDirectory());
    assertEquals(runtimeParameters, response.getRuntimeParameters());
    assertEquals(1, response.getJobExecutions().size());
    assertEquals(mapReduceJobId, response.getJobExecutions().get(0).getMapReduceJobId());
    assertEquals(MapReduceState.STARTED.toString(), response.getJobExecutions().get(0).getState());
}

Example 14 with MapReduceInfoResponseList

use of datawave.webservice.results.mr.MapReduceInfoResponseList in project datawave by NationalSecurityAgency.

the class MapReduceBean method cancel.

/**
 * Cancels any MapReduce jobs with the specified jobId and clears out the results directory
 *
 * @param jobId
 * @return {@code datawave.webservice.result.GenericResponse<Boolean>}
 * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user by specifying a chain of DNs of the identities to proxy
 * @RequestHeader X-ProxiedIssuersChain required when using X-ProxiedEntitiesChain, specify one issuer DN per subject DN listed in X-ProxiedEntitiesChain
 * @ResponseHeader X-OperationTimeInMS time spent on the server performing the operation, does not account for network or result serialization
 * @HTTP 200 success
 * @HTTP 404 if jobId is invalid or cannot be found
 * @HTTP 500 error killing the job
 */
@PUT
@Produces({ "application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff" })
@javax.ws.rs.Path("/{jobId}/cancel")
@GZIP
public GenericResponse<Boolean> cancel(@PathParam("jobId") String jobId) {
    GenericResponse<Boolean> response = new GenericResponse<>();
    // Find all potential running jobs
    MapReduceInfoResponseList list = mapReduceState.findById(jobId);
    List<String> jobIdsToKill = new ArrayList<>();
    // Should contain zero or one bulk result job
    if (list.getResults().isEmpty()) {
        NotFoundQueryException qe = new NotFoundQueryException(DatawaveErrorCode.NO_MAPREDUCE_OBJECT_MATCH);
        response.addException(qe);
        throw new NotFoundException(qe, response);
    } else if (list.getResults().size() > 1) {
        NotFoundQueryException qe = new NotFoundQueryException(DatawaveErrorCode.TOO_MANY_MAPREDUCE_OBJECT_MATCHES);
        response.addException(qe);
        throw new NotFoundException(qe, response);
    } else {
        MapReduceInfoResponse thisJob = list.getResults().get(0);
        // Get all the executions for this job
        String prevId = null;
        String prevState = null;
        // add it to the kill list
        for (JobExecution ex : thisJob.getJobExecutions()) {
            if (prevId != null) {
                if (prevState.equals(MapReduceState.STARTED.toString()) && !ex.getMapReduceJobId().equals(prevId))
                    jobIdsToKill.add(prevId);
            }
            prevId = ex.getMapReduceJobId();
            prevState = ex.getState();
        }
        // Get the last one
        if (MapReduceState.STARTED.toString().equals(prevState))
            jobIdsToKill.add(prevId);
        FileSystem hdfs = null;
        try {
            hdfs = getFS(thisJob.getHdfs(), response);
            Path resultsDir = new Path(thisJob.getResultsDirectory());
            hdfs.getConf().set("mapreduce.jobtracker.address", thisJob.getJobTracker());
            // Create a Job object
            try (JobClient job = new JobClient(new JobConf(hdfs.getConf()))) {
                for (String killId : jobIdsToKill) {
                    try {
                        JobID jid = JobID.forName(killId);
                        RunningJob rj = job.getJob(new org.apache.hadoop.mapred.JobID(jid.getJtIdentifier(), jid.getId()));
                        // job.getJob(jid);
                        if (null != rj)
                            rj.killJob();
                        else
                            mapReduceState.updateState(killId, MapReduceState.KILLED);
                    } catch (IOException | QueryException e) {
                        QueryException qe = new QueryException(DatawaveErrorCode.MAPREDUCE_JOB_KILL_ERROR, e, MessageFormat.format("job_id: {0}", killId));
                        log.error(qe);
                        response.addException(qe.getBottomQueryException());
                        throw new DatawaveWebApplicationException(qe, response);
                    }
                }
            }
            // Delete the contents of the results directory
            if (hdfs.exists(resultsDir) && !hdfs.delete(resultsDir, true)) {
                QueryException qe = new QueryException(DatawaveErrorCode.MAPRED_RESULTS_DELETE_ERROR, MessageFormat.format("directory: {0}", resultsDir.toString()));
                log.error(qe);
                response.addException(qe);
                throw new DatawaveWebApplicationException(qe, response);
            }
            response.setResult(true);
            return response;
        } catch (IOException e) {
            QueryException qe = new QueryException(DatawaveErrorCode.JOBTRACKER_CONNECTION_ERROR, e, MessageFormat.format("JobTracker: {0}", thisJob.getJobTracker()));
            log.error(qe);
            response.addException(qe);
            throw new DatawaveWebApplicationException(qe, response);
        } finally {
            if (null != hdfs) {
                try {
                    hdfs.close();
                } catch (IOException e) {
                    log.error("Error closing HDFS client", e);
                }
            }
        }
    }
}