use of datawave.webservice.query.exception.QueryException in project datawave by NationalSecurityAgency.
the class FileSortedSet method first.
@Override
public E first() {
boolean gotFirst = false;
E first = null;
if (persisted) {
try (SortedSetInputStream<E> stream = getBoundedFileHandler().getInputStream(getStart(), getEnd())) {
first = stream.readObject();
gotFirst = true;
} catch (Exception e) {
QueryException qe = new QueryException(DatawaveErrorCode.FETCH_FIRST_ELEMENT_ERROR, e);
throw (new IllegalStateException(qe));
}
} else if (!set.isEmpty()) {
first = set.first();
gotFirst = true;
}
if (!gotFirst) {
QueryException qe = new QueryException(DatawaveErrorCode.FETCH_FIRST_ELEMENT_ERROR);
throw (NoSuchElementException) (new NoSuchElementException().initCause(qe));
} else {
return first;
}
}
use of datawave.webservice.query.exception.QueryException in project datawave by NationalSecurityAgency.
the class ShardQueryLogic method setupQuery.
@Override
public void setupQuery(GenericQueryConfiguration genericConfig) throws Exception {
if (!ShardQueryConfiguration.class.isAssignableFrom(genericConfig.getClass())) {
throw new QueryException("Did not receive a ShardQueryConfiguration instance!!");
}
ShardQueryConfiguration config = (ShardQueryConfiguration) genericConfig;
final QueryStopwatch timers = config.getTimers();
TraceStopwatch stopwatch = timers.newStartedStopwatch("ShardQueryLogic - Setup Query");
// Ensure we have all of the information needed to run a query
if (!config.canRunQuery()) {
log.warn("The given query '" + config + "' could not be run, most likely due to not matching any records in the global index.");
// Stub out an iterator to correctly present "no results"
this.iterator = new Iterator<Map.Entry<Key, Value>>() {
@Override
public boolean hasNext() {
return false;
}
@Override
public Map.Entry<Key, Value> next() {
return null;
}
@Override
public void remove() {
return;
}
};
this.scanner = null;
stopwatch.stop();
log.info(getStopwatchHeader(config));
List<String> timings = timers.summarizeAsList();
for (String timing : timings) {
log.info(timing);
}
return;
}
// Instantiate the scheduler for the queries
this.scheduler = getScheduler(config, scannerFactory);
this.scanner = null;
this.iterator = this.scheduler.iterator();
if (!config.isSortedUIDs()) {
this.iterator = new DedupingIterator(this.iterator);
}
stopwatch.stop();
log.info(getStopwatchHeader(config));
List<String> timings = timers.summarizeAsList();
for (String timing : timings) {
log.info(timing);
}
}
use of datawave.webservice.query.exception.QueryException in project datawave by NationalSecurityAgency.
the class ShardQueryLogic method loadQueryParameters.
protected void loadQueryParameters(ShardQueryConfiguration config, Query settings) throws QueryException {
TraceStopwatch stopwatch = config.getTimers().newStartedStopwatch("ShardQueryLogic - Parse query parameters");
boolean rawDataOnly = false;
String rawDataOnlyStr = settings.findParameter(QueryParameters.RAW_DATA_ONLY).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(rawDataOnlyStr)) {
rawDataOnly = Boolean.valueOf(rawDataOnlyStr);
// note that if any of these other options are set, then it overrides the settings here
if (rawDataOnly) {
// set the grouping context to trye to ensure we get the full field names
this.setIncludeGroupingContext(true);
config.setIncludeGroupingContext(true);
// set the hierarchy fields to false as they are generated fields
this.setIncludeHierarchyFields(false);
config.setIncludeHierarchyFields(false);
// set the datatype field to false as it is a generated field
this.setIncludeDataTypeAsField(false);
config.setIncludeDataTypeAsField(false);
// do not include the record id
this.setIncludeRecordId(false);
config.setIncludeRecordId(false);
// set the hit list to false as it is a generated field
this.setHitList(false);
config.setHitList(false);
// set the raw types to true to avoid any type transformations of the values
config.setRawTypes(true);
// do not filter masked values
this.setFilterMaskedValues(false);
config.setFilterMaskedValues(false);
// do not reduce the response
this.setReducedResponse(false);
config.setReducedResponse(false);
// clear the content field names to prevent content field transformations (see DocumentTransformer)
this.setContentFieldNames(Collections.EMPTY_LIST);
// clear the model name to avoid field name translations
this.setModelName(null);
config.setModelName(null);
}
}
// Get the datatype set if specified
String typeList = settings.findParameter(QueryParameters.DATATYPE_FILTER_SET).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(typeList)) {
HashSet<String> typeFilter = new HashSet<>();
typeFilter.addAll(Arrays.asList(StringUtils.split(typeList, Constants.PARAM_VALUE_SEP)));
if (log.isDebugEnabled()) {
log.debug("Type Filter: " + typeFilter);
}
config.setDatatypeFilter(typeFilter);
}
// Get the list of fields to project up the stack. May be null.
String projectFields = settings.findParameter(QueryParameters.RETURN_FIELDS).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(projectFields)) {
List<String> projectFieldsList = Arrays.asList(StringUtils.split(projectFields, Constants.PARAM_VALUE_SEP));
// Only set the projection fields if we were actually given some
if (!projectFieldsList.isEmpty()) {
config.setProjectFields(new HashSet<>(projectFieldsList));
if (log.isDebugEnabled()) {
final int maxLen = 100;
// Trim down the projection if it's stupid long
projectFields = maxLen < projectFields.length() ? projectFields.substring(0, maxLen) + "[TRUNCATED]" : projectFields;
log.debug("Projection fields: " + projectFields);
}
}
}
// if the TRANFORM_CONTENT_TO_UID is false, then unset the list of content field names preventing the DocumentTransformer from
// transforming them.
String transformContentStr = settings.findParameter(QueryParameters.TRANFORM_CONTENT_TO_UID).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(transformContentStr)) {
if (!Boolean.valueOf(transformContentStr)) {
setContentFieldNames(Collections.EMPTY_LIST);
}
}
// Get the list of blacklisted fields. May be null.
String tBlacklistedFields = settings.findParameter(QueryParameters.BLACKLISTED_FIELDS).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(tBlacklistedFields)) {
List<String> blacklistedFieldsList = Arrays.asList(StringUtils.split(tBlacklistedFields, Constants.PARAM_VALUE_SEP));
// Only set the blacklisted fields if we were actually given some
if (!blacklistedFieldsList.isEmpty()) {
if (!config.getProjectFields().isEmpty()) {
throw new QueryException("Whitelist and blacklist projection options are mutually exclusive");
}
config.setBlacklistedFields(new HashSet<>(blacklistedFieldsList));
if (log.isDebugEnabled()) {
log.debug("Blacklisted fields: " + tBlacklistedFields);
}
}
}
// Get the LIMIT_FIELDS parameter if given
String limitFields = settings.findParameter(QueryParameters.LIMIT_FIELDS).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(limitFields)) {
List<String> limitFieldsList = Arrays.asList(StringUtils.split(limitFields, Constants.PARAM_VALUE_SEP));
// Only set the limit fields if we were actually given some
if (!limitFieldsList.isEmpty()) {
config.setLimitFields(new HashSet<>(limitFieldsList));
}
}
String limitFieldsPreQueryEvaluation = settings.findParameter(QueryOptions.LIMIT_FIELDS_PRE_QUERY_EVALUATION).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(limitFieldsPreQueryEvaluation)) {
Boolean limitFieldsPreQueryEvaluationValue = Boolean.parseBoolean(limitFieldsPreQueryEvaluation);
this.setLimitFieldsPreQueryEvaluation(limitFieldsPreQueryEvaluationValue);
config.setLimitFieldsPreQueryEvaluation(limitFieldsPreQueryEvaluationValue);
}
String limitFieldsField = settings.findParameter(QueryOptions.LIMIT_FIELDS_FIELD).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(limitFieldsField)) {
this.setLimitFieldsField(limitFieldsField);
config.setLimitFieldsField(limitFieldsField);
}
// Get the GROUP_FIELDS parameter if given
String groupFields = settings.findParameter(QueryParameters.GROUP_FIELDS).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(groupFields)) {
List<String> groupFieldsList = Arrays.asList(StringUtils.split(groupFields, Constants.PARAM_VALUE_SEP));
// Only set the group fields if we were actually given some
if (!groupFieldsList.isEmpty()) {
this.setGroupFields(new HashSet<>(groupFieldsList));
config.setGroupFields(new HashSet<>(groupFieldsList));
config.setProjectFields(new HashSet<>(groupFieldsList));
}
}
String groupFieldsBatchSizeString = settings.findParameter(QueryParameters.GROUP_FIELDS_BATCH_SIZE).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(groupFieldsBatchSizeString)) {
int groupFieldsBatchSize = Integer.parseInt(groupFieldsBatchSizeString);
this.setGroupFieldsBatchSize(groupFieldsBatchSize);
config.setGroupFieldsBatchSize(groupFieldsBatchSize);
}
// Get the UNIQUE_FIELDS parameter if given
String uniqueFieldsParam = settings.findParameter(QueryParameters.UNIQUE_FIELDS).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(uniqueFieldsParam)) {
UniqueFields uniqueFields = UniqueFields.from(uniqueFieldsParam);
// Only set the unique fields if we were actually given some
if (!uniqueFields.isEmpty()) {
this.setUniqueFields(uniqueFields);
config.setUniqueFields(uniqueFields);
}
}
// Get the HIT_LIST parameter if given
String hitListString = settings.findParameter(QueryParameters.HIT_LIST).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(hitListString)) {
Boolean hitListBool = Boolean.parseBoolean(hitListString);
config.setHitList(hitListBool);
}
// Get the BYPASS_ACCUMULO parameter if given
String bypassAccumuloString = settings.findParameter(BYPASS_ACCUMULO).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(bypassAccumuloString)) {
Boolean bypassAccumuloBool = Boolean.parseBoolean(bypassAccumuloString);
config.setBypassAccumulo(bypassAccumuloBool);
}
// Get the DATE_INDEX_TIME_TRAVEL parameter if given
String dateIndexTimeTravelString = settings.findParameter(QueryOptions.DATE_INDEX_TIME_TRAVEL).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(dateIndexTimeTravelString)) {
Boolean dateIndexTimeTravel = Boolean.parseBoolean(dateIndexTimeTravelString);
config.setDateIndexTimeTravel(dateIndexTimeTravel);
}
// get the RAW_TYPES parameter if given
String rawTypesString = settings.findParameter(QueryParameters.RAW_TYPES).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(rawTypesString)) {
Boolean rawTypesBool = Boolean.parseBoolean(rawTypesString);
config.setRawTypes(rawTypesBool);
}
// Get the FILTER_MASKED_VALUES spring setting
String filterMaskedValuesStr = settings.findParameter(QueryParameters.FILTER_MASKED_VALUES).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(filterMaskedValuesStr)) {
Boolean filterMaskedValuesBool = Boolean.parseBoolean(filterMaskedValuesStr);
this.setFilterMaskedValues(filterMaskedValuesBool);
config.setFilterMaskedValues(filterMaskedValuesBool);
}
// Get the INCLUDE_DATATYPE_AS_FIELD spring setting
String includeDatatypeAsFieldStr = settings.findParameter(QueryParameters.INCLUDE_DATATYPE_AS_FIELD).getParameterValue().trim();
if (((org.apache.commons.lang.StringUtils.isNotBlank(includeDatatypeAsFieldStr) && Boolean.valueOf(includeDatatypeAsFieldStr))) || (this.getIncludeDataTypeAsField() && !rawDataOnly)) {
config.setIncludeDataTypeAsField(true);
}
// Get the INCLUDE_RECORD_ID spring setting
String includeRecordIdStr = settings.findParameter(QueryParameters.INCLUDE_RECORD_ID).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(includeRecordIdStr)) {
boolean includeRecordIdBool = Boolean.parseBoolean(includeRecordIdStr) && !rawDataOnly;
this.setIncludeRecordId(includeRecordIdBool);
config.setIncludeRecordId(includeRecordIdBool);
}
// Get the INCLUDE_HIERARCHY_FIELDS spring setting
String includeHierarchyFieldsStr = settings.findParameter(QueryParameters.INCLUDE_HIERARCHY_FIELDS).getParameterValue().trim();
if (((org.apache.commons.lang.StringUtils.isNotBlank(includeHierarchyFieldsStr) && Boolean.valueOf(includeHierarchyFieldsStr))) || (this.getIncludeHierarchyFields() && !rawDataOnly)) {
config.setIncludeHierarchyFields(true);
final Map<String, String> options = this.getHierarchyFieldOptions();
config.setHierarchyFieldOptions(options);
}
// Get the query profile to allow us to select the tune profile of the query
String queryProfile = settings.findParameter(QueryParameters.QUERY_PROFILE).getParameterValue().trim();
if ((org.apache.commons.lang.StringUtils.isNotBlank(queryProfile))) {
selectedProfile = configuredProfiles.get(queryProfile);
if (null == selectedProfile) {
throw new QueryException(QueryParameters.QUERY_PROFILE + " has been specified but " + queryProfile + " is not a selectable profile");
}
}
// Get the include.grouping.context = true/false spring setting
String includeGroupingContextStr = settings.findParameter(QueryParameters.INCLUDE_GROUPING_CONTEXT).getParameterValue().trim();
if (((org.apache.commons.lang.StringUtils.isNotBlank(includeGroupingContextStr) && Boolean.valueOf(includeGroupingContextStr))) || (this.getIncludeGroupingContext() && !rawDataOnly)) {
config.setIncludeGroupingContext(true);
}
// Check if the default modelName and modelTableNames have been overridden by custom parameters.
String parameterModelName = settings.findParameter(QueryParameters.PARAMETER_MODEL_NAME).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(parameterModelName)) {
this.setModelName(parameterModelName);
}
config.setModelName(this.getModelName());
String parameterModelTableName = settings.findParameter(QueryParameters.PARAMETER_MODEL_TABLE_NAME).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(parameterModelTableName)) {
this.setModelTableName(parameterModelTableName);
}
if (null != config.getModelName() && null == config.getModelTableName()) {
throw new IllegalArgumentException(QueryParameters.PARAMETER_MODEL_NAME + " has been specified but " + QueryParameters.PARAMETER_MODEL_TABLE_NAME + " is missing. Both are required to use a model");
}
configureDocumentAggregation(settings);
config.setLimitTermExpansionToModel(this.isExpansionLimitedToModelContents());
String reducedResponseStr = settings.findParameter(QueryOptions.REDUCED_RESPONSE).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(reducedResponseStr)) {
Boolean reducedResponseValue = Boolean.parseBoolean(reducedResponseStr);
this.setReducedResponse(reducedResponseValue);
config.setReducedResponse(reducedResponseValue);
}
final String postProcessingClasses = settings.findParameter(QueryOptions.POSTPROCESSING_CLASSES).getParameterValue().trim();
final String postProcessingOptions = settings.findParameter(QueryOptions.POSTPROCESSING_OPTIONS).getParameterValue().trim();
// build the post p
if (org.apache.commons.lang.StringUtils.isNotBlank(postProcessingClasses)) {
List<String> filterClasses = config.getFilterClassNames();
if (null == filterClasses) {
filterClasses = new ArrayList<>();
}
for (String fClassName : StringUtils.splitIterable(postProcessingClasses, ',', true)) {
filterClasses.add(fClassName);
}
config.setFilterClassNames(filterClasses);
final Map<String, String> options = this.getFilterOptions();
if (null != options) {
config.putFilterOptions(options);
}
if (org.apache.commons.lang.StringUtils.isNotBlank(postProcessingOptions)) {
for (String filterOptionStr : StringUtils.splitIterable(postProcessingOptions, ',', true)) {
if (org.apache.commons.lang.StringUtils.isNotBlank(filterOptionStr)) {
final String filterValueString = settings.findParameter(filterOptionStr).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(filterValueString)) {
config.putFilterOptions(filterOptionStr, filterValueString);
}
}
}
}
}
String tCompressServerSideResults = settings.findParameter(QueryOptions.COMPRESS_SERVER_SIDE_RESULTS).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(tCompressServerSideResults)) {
boolean compress = Boolean.parseBoolean(tCompressServerSideResults);
config.setCompressServerSideResults(compress);
}
// Configure index-only filter functions to be enabled if not already set to such a state
config.setIndexOnlyFilterFunctionsEnabled(this.isIndexOnlyFilterFunctionsEnabled());
// Set the ReturnType for Documents coming out of the iterator stack
config.setReturnType(DocumentSerialization.getReturnType(settings));
QueryLogicTransformer transformer = getTransformer(settings);
if (transformer instanceof WritesQueryMetrics) {
String logTimingDetailsStr = settings.findParameter(QueryOptions.LOG_TIMING_DETAILS).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(logTimingDetailsStr)) {
setLogTimingDetails(Boolean.valueOf(logTimingDetailsStr));
}
if (getLogTimingDetails()) {
// we have to collect the timing details on the iterator stack in order to log them
setCollectTimingDetails(true);
} else {
String collectTimingDetailsStr = settings.findParameter(QueryOptions.COLLECT_TIMING_DETAILS).getParameterValue().trim();
if (org.apache.commons.lang.StringUtils.isNotBlank(collectTimingDetailsStr)) {
setCollectTimingDetails(Boolean.valueOf(collectTimingDetailsStr));
}
}
} else {
// if the transformer can not process the timing metrics, then turn them off
setLogTimingDetails(false);
setCollectTimingDetails(false);
}
stopwatch.stop();
if (null != selectedProfile) {
selectedProfile.configure(this);
selectedProfile.configure(config);
selectedProfile.configure(planner);
}
}
use of datawave.webservice.query.exception.QueryException in project datawave by NationalSecurityAgency.
the class ContentQueryTable method setupQuery.
@Override
public void setupQuery(GenericQueryConfiguration genericConfig) throws Exception {
if (!genericConfig.getClass().getName().equals(ContentQueryConfiguration.class.getName())) {
throw new QueryException("Did not receive a ContentQueryConfiguration instance!!");
}
final ContentQueryConfiguration config = (ContentQueryConfiguration) genericConfig;
try {
final BatchScanner scanner = this.scannerFactory.newScanner(config.getTableName(), config.getAuthorizations(), this.queryThreads, config.getQuery());
scanner.setRanges(config.getRanges());
if (null != this.viewName) {
final IteratorSetting cfg = new IteratorSetting(50, RegExFilter.class);
cfg.addOption(RegExFilter.COLQ_REGEX, this.viewName);
scanner.addScanIterator(cfg);
}
this.iterator = scanner.iterator();
this.scanner = scanner;
} catch (TableNotFoundException e) {
throw new RuntimeException("Table not found: " + this.getTableName(), e);
}
}
use of datawave.webservice.query.exception.QueryException in project datawave by NationalSecurityAgency.
the class CachedResultsBean method reset.
/**
* Re-allocate resources for these cached results and reset paging to the beginning
*
* @param queryId
* user defined id for this query
*
* @return datawave.webservice.result.CachedResultsResponse
* @RequestHeader X-ProxiedEntitiesChain use when proxying request for user by specifying a chain of DNs of the identities to proxy
* @RequestHeader X-ProxiedIssuersChain required when using X-ProxiedEntitiesChain, specify one issuer DN per subject DN listed in X-ProxiedEntitiesChain
* @ResponseHeader query-session-id this header and value will be in the Set-Cookie header, subsequent calls for this session will need to supply the
* query-session-id header in the request in a Cookie header or as a query parameter
* @ResponseHeader X-OperationTimeInMS time spent on the server performing the operation, does not account for network or result serialization
*
* @HTTP 200 success
* @HTTP 500 internal server error
*/
@PUT
@Produces({ "application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml" })
@javax.ws.rs.Path("/{queryId}/reset")
@Interceptors({ RequiredInterceptor.class, ResponseInterceptor.class })
@GenerateQuerySessionId(cookieBasePath = "/DataWave/CachedResults/")
@Timed(name = "dw.cachedr.close", absolute = true)
public CachedResultsResponse reset(@PathParam("queryId") @Required("queryId") String queryId) {
CreateQuerySessionIDFilter.QUERY_ID.set(null);
CachedResultsResponse response = new CachedResultsResponse();
// Find out who/what called this method
Principal p = ctx.getCallerPrincipal();
String owner = getOwnerFromPrincipal(p);
try {
CachedRunningQuery crq = null;
try {
// Get the CachedRunningQuery object from the cache
try {
crq = retrieve(queryId, owner);
} catch (IOException e) {
throw new PreConditionFailedQueryException(DatawaveErrorCode.CACHED_RESULTS_IMPORT_ERROR);
}
if (null == crq)
throw new PreConditionFailedQueryException(DatawaveErrorCode.QUERY_NOT_CACHED);
if (!crq.getUser().equals(owner)) {
throw new UnauthorizedQueryException(DatawaveErrorCode.QUERY_OWNER_MISMATCH, MessageFormat.format("{0} != {1}", crq.getUser(), owner));
}
synchronized (crq) {
if (crq.isActivated() == true) {
crq.closeConnection(log);
}
Connection connection = ds.getConnection();
String logicName = crq.getQueryLogicName();
QueryLogic<?> queryLogic = queryFactory.getQueryLogic(logicName, p);
crq.activate(connection, queryLogic);
response.setQueryId(crq.getQueryId());
response.setOriginalQueryId(crq.getOriginalQueryId());
response.setViewName(crq.getView());
response.setAlias(crq.getAlias());
response.setTotalRows(crq.getTotalRows());
}
CreateQuerySessionIDFilter.QUERY_ID.set(queryId);
return response;
} finally {
// Push metrics
if (null != crq && crq.getQueryLogic().getCollectQueryMetrics() == true) {
try {
metrics.updateMetric(crq.getMetric());
} catch (Exception e1) {
log.error("Error updating metrics", e1);
}
}
}
} catch (Exception e) {
response.addMessage(e.getMessage());
QueryException qe = new QueryException(DatawaveErrorCode.RESET_CALL_ERROR, e);
log.error(qe);
response.addException(qe.getBottomQueryException());
int statusCode = qe.getBottomQueryException().getStatusCode();
throw new DatawaveWebApplicationException(qe, response, statusCode);
}
}
Aggregations