use of datawave.webservice.query.cache.ResultsPage in project datawave by NationalSecurityAgency.
the class BulkResultsTableOutputMapper method map.
@Override
protected void map(Key key, Value value, org.apache.hadoop.mapreduce.Mapper<Key, Value, Text, Mutation>.Context context) throws IOException, InterruptedException {
entries.clear();
entries.put(key, value);
for (Entry<Key, Value> entry : entries.entrySet()) {
try {
Object o = t.transform(entry);
BaseQueryResponse response = t.createResponse(new ResultsPage(Collections.singletonList(o)));
Class<? extends BaseQueryResponse> responseClass = null;
try {
responseClass = getResponseClass(response.getClass().getName());
} catch (ClassNotFoundException e) {
throw new RuntimeException("Unable to find response class: " + response.getClass().getName(), e);
}
try {
Value val = BulkResultsFileOutputMapper.serializeResponse(responseClass, response, this.format);
// Write out the original key and the new value.
Mutation m = new Mutation(key.getRow());
m.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), val);
context.write(this.tableName, m);
} catch (Exception e) {
throw new RuntimeException("Unable to serialize response of class: " + response.getClass().getName(), e);
}
context.progress();
} catch (EmptyObjectException e) {
// not yet done, so continue fetching next
}
}
}
use of datawave.webservice.query.cache.ResultsPage in project datawave by NationalSecurityAgency.
the class QueryExecutorBean method execute.
/**
* @param logicName
* @param queryParameters
*
* @return {@code datawave.webservice.result.GenericResponse<String>}
* @RequestHeader X-ProxiedEntitiesChain use when proxying request for user, by specifying a chain of DNs of the identities to proxy
* @RequestHeader X-ProxiedIssuersChain required when using X-ProxiedEntitiesChain, specify one issuer DN per subject DN listed in X-ProxiedEntitiesChain
* @ResponseHeader query-session-id this header and value will be in the Set-Cookie header, subsequent calls for this session will need to supply the
* query-session-id header in the request in a Cookie header or as a query parameter
* @ResponseHeader X-OperationTimeInMS time spent on the server performing the operation, does not account for network or result serialization
* @ResponseHeader X-Partial-Results true if the page contains less than the requested number of results
*
* @HTTP 200 success
* @HTTP 204 success and no results
* @HTTP 400 invalid or missing parameter
* @HTTP 500 internal server error
*/
@POST
@Produces("*/*")
@Path("/{logicName}/execute")
@GZIP
@Interceptors({ ResponseInterceptor.class, RequiredInterceptor.class })
@TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
@Override
@Timed(name = "dw.query.executeQuery", absolute = true)
public StreamingOutput execute(@PathParam("logicName") String logicName, MultivaluedMap<String, String> queryParameters, @Context HttpHeaders httpHeaders) {
/**
* This method captures the metrics on the query instead of doing it in the QueryMetricsEnrichmentInterceptor. The ExecuteStreamingOutputResponse class
* is returned from this method and executed in the JAX-RS layer. It updates the metrics which are then updated on each call to the _next method.
*/
Collection<String> proxyServers = null;
Principal p = ctx.getCallerPrincipal();
DatawavePrincipal dp;
if (p instanceof DatawavePrincipal) {
dp = (DatawavePrincipal) p;
proxyServers = dp.getProxyServers();
}
final MediaType PB_MEDIA_TYPE = new MediaType("application", "x-protobuf");
final MediaType YAML_MEDIA_TYPE = new MediaType("application", "x-yaml");
final VoidResponse response = new VoidResponse();
// HttpHeaders.getAcceptableMediaTypes returns a priority sorted list of acceptable response types.
// Find the first one in the list that we support.
MediaType responseType = null;
for (MediaType type : httpHeaders.getAcceptableMediaTypes()) {
if (type.equals(MediaType.APPLICATION_XML_TYPE) || type.equals(MediaType.APPLICATION_JSON_TYPE) || type.equals(PB_MEDIA_TYPE) || type.equals(YAML_MEDIA_TYPE)) {
responseType = type;
break;
}
}
if (null == responseType) {
QueryException qe = new QueryException(DatawaveErrorCode.UNSUPPORTED_MEDIA_TYPE);
response.setHasResults(false);
response.addException(qe);
throw new DatawaveWebApplicationException(qe, response, MediaType.APPLICATION_XML_TYPE);
}
// reference query necessary to avoid NPEs in getting the Transformer and BaseResponse
Query q = new QueryImpl();
Date now = new Date();
q.setBeginDate(now);
q.setEndDate(now);
q.setExpirationDate(now);
q.setQuery("test");
q.setQueryAuthorizations("ALL");
ResultsPage emptyList = new ResultsPage();
// Find the response class
Class<?> responseClass;
try {
QueryLogic<?> l = queryLogicFactory.getQueryLogic(logicName, p);
QueryLogicTransformer t = l.getTransformer(q);
BaseResponse refResponse = t.createResponse(emptyList);
responseClass = refResponse.getClass();
} catch (Exception e) {
QueryException qe = new QueryException(DatawaveErrorCode.QUERY_TRANSFORM_ERROR, e);
log.error(qe, e);
response.setHasResults(false);
response.addException(qe.getBottomQueryException());
int statusCode = qe.getBottomQueryException().getStatusCode();
throw new DatawaveWebApplicationException(qe, response, statusCode, MediaType.APPLICATION_XML_TYPE);
}
SerializationType s;
if (responseType.equals(MediaType.APPLICATION_XML_TYPE)) {
s = SerializationType.XML;
} else if (responseType.equals(MediaType.APPLICATION_JSON_TYPE)) {
s = SerializationType.JSON;
} else if (responseType.equals(PB_MEDIA_TYPE)) {
if (!(Message.class.isAssignableFrom(responseClass))) {
QueryException qe = new QueryException(DatawaveErrorCode.BAD_RESPONSE_CLASS, MessageFormat.format("Response class: {0}", responseClass));
response.setHasResults(false);
response.addException(qe);
throw new DatawaveWebApplicationException(qe, response, MediaType.APPLICATION_XML_TYPE);
}
s = SerializationType.PB;
} else if (responseType.equals(YAML_MEDIA_TYPE)) {
if (!(Message.class.isAssignableFrom(responseClass))) {
QueryException qe = new QueryException(DatawaveErrorCode.BAD_RESPONSE_CLASS, MessageFormat.format("Response class: {0}", responseClass));
response.setHasResults(false);
response.addException(qe);
throw new DatawaveWebApplicationException(qe, response, MediaType.APPLICATION_XML_TYPE);
}
s = SerializationType.YAML;
} else {
QueryException qe = new QueryException(DatawaveErrorCode.INVALID_FORMAT, MessageFormat.format("format: {0}", responseType.toString()));
response.setHasResults(false);
response.addException(qe);
throw new DatawaveWebApplicationException(qe, response, MediaType.APPLICATION_XML_TYPE);
}
long start = System.nanoTime();
GenericResponse<String> createResponse = null;
try {
createResponse = this.createQuery(logicName, queryParameters, httpHeaders);
} catch (Throwable t) {
if (t instanceof DatawaveWebApplicationException) {
QueryException qe = (QueryException) ((DatawaveWebApplicationException) t).getCause();
response.setHasResults(false);
response.addException(qe.getBottomQueryException());
int statusCode = qe.getBottomQueryException().getStatusCode();
throw new DatawaveWebApplicationException(qe, response, statusCode, MediaType.APPLICATION_XML_TYPE);
} else {
throw t;
}
}
long createCallTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
final String queryId = createResponse.getResult();
// We created the query and put into cache, get the RunningQuery object
final RunningQuery rq = queryCache.get(queryId);
rq.getMetric().setCreateCallTime(createCallTime);
final Collection<String> proxies = proxyServers;
final SerializationType serializationType = s;
final Class<?> queryResponseClass = responseClass;
return new ExecuteStreamingOutputResponse(queryId, queryResponseClass, response, rq, serializationType, proxies);
}
use of datawave.webservice.query.cache.ResultsPage in project datawave by NationalSecurityAgency.
the class CompositeQueryLogicTest method testQueryLogic.
@Test
public // testQueryLogic with max.results.override not set
void testQueryLogic() throws Exception {
Logger.getLogger(CompositeQueryLogic.class).setLevel(Level.TRACE);
Logger.getLogger(CompositeQueryLogicResults.class).setLevel(Level.TRACE);
Logger.getLogger(CompositeQueryLogicTransformer.class).setLevel(Level.TRACE);
List<BaseQueryLogic<?>> logics = new ArrayList<>();
TestQueryLogic logic1 = new TestQueryLogic();
TestQueryLogic2 logic2 = new TestQueryLogic2();
logics.add(logic1);
logics.add(logic2);
logic1.getData().put(key1, value1);
logic1.getData().put(key2, value2);
logic2.getData().put(key3, value3);
logic2.getData().put(key4, value4);
logic1.getData().put(key5, value5);
logic1.getData().put(key6, value6);
logic2.getData().put(key7, value7);
logic2.getData().put(key8, value8);
QueryImpl settings = new QueryImpl();
settings.setPagesize(100);
settings.setQueryAuthorizations(auths.toString());
settings.setQuery("FOO == 'BAR'");
settings.setParameters(new HashSet<>());
settings.setId(UUID.randomUUID());
CompositeQueryLogic c = new CompositeQueryLogic();
// max.results.override is set to -1 when it is not passed in as it is an optional paramter
logic1.setMaxResults(-1);
logic2.setMaxResults(-1);
/**
* RunningQuery.setupConnection()
*/
c.setQueryLogics(logics);
c.initialize((Connector) null, (Query) settings, Collections.singleton(auths));
c.setupQuery(null);
TransformIterator iter = c.getTransformIterator((Query) settings);
/**
* RunningQuery.next() - iterate over results coming from tablet server through the TransformIterator to turn them into the objects.
*/
List<Object> results = new ArrayList<>();
while (iter.hasNext()) {
Object o = iter.next();
if (null == o)
break;
Assert.assertTrue(o instanceof TestQueryResponse);
results.add((TestQueryResponse) o);
}
Assert.assertEquals(8, results.size());
ResultsPage page = new ResultsPage(results, Status.COMPLETE);
/**
* QueryExecutorBean.next() - transform list of objects into JAXB response
*/
TestQueryResponseList response = (TestQueryResponseList) c.getTransformer((Query) settings).createResponse(page);
Assert.assertEquals(8, response.getResponses().size());
for (TestQueryResponse r : response.getResponses()) {
Assert.assertNotNull(r);
}
c.close();
}
use of datawave.webservice.query.cache.ResultsPage in project datawave by NationalSecurityAgency.
the class CompositeQueryLogicTest method testQueryLogicWithMaxResultsOverride.
@Test
public // testQueryLogic with max.results.override is set
void testQueryLogicWithMaxResultsOverride() throws Exception {
Logger.getLogger(CompositeQueryLogic.class).setLevel(Level.TRACE);
Logger.getLogger(CompositeQueryLogicResults.class).setLevel(Level.TRACE);
Logger.getLogger(CompositeQueryLogicTransformer.class).setLevel(Level.TRACE);
List<BaseQueryLogic<?>> logics = new ArrayList<>();
TestQueryLogic logic1 = new TestQueryLogic();
TestQueryLogic2 logic2 = new TestQueryLogic2();
logics.add(logic1);
logics.add(logic2);
logic1.getData().put(key1, value1);
logic1.getData().put(key2, value2);
logic2.getData().put(key3, value3);
logic2.getData().put(key4, value4);
logic1.getData().put(key5, value5);
logic1.getData().put(key6, value6);
logic2.getData().put(key7, value7);
logic2.getData().put(key8, value8);
QueryImpl settings = new QueryImpl();
settings.setPagesize(100);
settings.setQueryAuthorizations(auths.toString());
settings.setQuery("FOO == 'BAR'");
settings.setParameters(new HashSet<>());
settings.setId(UUID.randomUUID());
CompositeQueryLogic c = new CompositeQueryLogic();
// max.results.override is set to -1 when it is not passed in as it is an optional parameter
logic1.setMaxResults(0);
logic2.setMaxResults(4);
/**
* RunningQuery.setupConnection()
*/
c.setQueryLogics(logics);
c.initialize((Connector) null, (Query) settings, Collections.singleton(auths));
c.setupQuery(null);
TransformIterator iter = c.getTransformIterator((Query) settings);
/**
* RunningQuery.next() - iterate over results coming from tablet server through the TransformIterator to turn them into the objects.
*/
List<Object> results = new ArrayList<>();
while (iter.hasNext()) {
Object o = iter.next();
if (null == o)
break;
Assert.assertTrue(o instanceof TestQueryResponse);
results.add((TestQueryResponse) o);
}
Assert.assertEquals(4, results.size());
ResultsPage page = new ResultsPage(results, Status.COMPLETE);
/**
* QueryExecutorBean.next() - transform list of objects into JAXB response
*/
TestQueryResponseList response = (TestQueryResponseList) c.getTransformer((Query) settings).createResponse(page);
Assert.assertEquals(4, response.getResponses().size());
for (TestQueryResponse r : response.getResponses()) {
Assert.assertNotNull(r);
}
c.close();
}
use of datawave.webservice.query.cache.ResultsPage in project datawave by NationalSecurityAgency.
the class CompositeQueryLogicTest method testQueryLogicNoDataLogic1.
@Test
public void testQueryLogicNoDataLogic1() throws Exception {
List<BaseQueryLogic<?>> logics = new ArrayList<>();
TestQueryLogic logic1 = new TestQueryLogic();
TestQueryLogic2 logic2 = new TestQueryLogic2();
logics.add(logic1);
logics.add(logic2);
logic2.getData().put(key1, value1);
logic2.getData().put(key2, value2);
logic2.getData().put(key3, value3);
logic2.getData().put(key4, value4);
logic2.getData().put(key5, value5);
logic2.getData().put(key6, value6);
logic2.getData().put(key7, value7);
logic2.getData().put(key8, value8);
QueryImpl settings = new QueryImpl();
settings.setPagesize(100);
settings.setQueryAuthorizations(auths.serialize());
settings.setQuery("FOO == 'BAR'");
settings.setParameters(new HashSet<>());
settings.setId(UUID.randomUUID());
CompositeQueryLogic c = new CompositeQueryLogic();
// max.results.override is set to -1 when it is not passed in as it is an optional paramter
logic2.setMaxResults(-1);
logic1.setMaxResults(-1);
/**
* RunningQuery.setupConnection()
*/
c.setQueryLogics(logics);
c.initialize((Connector) null, (Query) settings, Collections.singleton(auths));
c.setupQuery(null);
TransformIterator iter = c.getTransformIterator((Query) settings);
/**
* RunningQuery.next() - iterate over results coming from tablet server through the TransformIterator to turn them into the objects.
*/
List<Object> results = new ArrayList<>();
while (iter.hasNext()) {
Object o = iter.next();
if (null == o)
break;
Assert.assertTrue(o instanceof TestQueryResponse);
results.add((TestQueryResponse) o);
}
Assert.assertEquals(8, results.size());
ResultsPage page = new ResultsPage(results, Status.COMPLETE);
/**
* QueryExecutorBean.next() - transform list of objects into JAXB response
*/
TestQueryResponseList response = (TestQueryResponseList) c.getTransformer((Query) settings).createResponse(page);
Assert.assertEquals(8, response.getResponses().size());
for (TestQueryResponse r : response.getResponses()) {
Assert.assertNotNull(r);
}
c.close();
}
Aggregations