use of datawave.webservice.query.exception.QueryException in project datawave by NationalSecurityAgency.
the class ExtendedQueryExecutorBeanTest method testDefineQuery_userNotInAllowedDNs.
@Test
public void testDefineQuery_userNotInAllowedDNs() throws Exception {
// Set local test input
String queryLogicName = "queryLogicName";
String query = "query";
String queryName = "queryName";
String queryVisibility = "A&B";
long currentTime = System.currentTimeMillis();
Date beginDate = new Date(currentTime - 5000);
Date endDate = new Date(currentTime - 1000);
String queryAuthorizations = "AUTH_1";
Date expirationDate = new Date(currentTime + 9999);
int pagesize = 1000;
int pageTimeout = -1;
Long maxResultsOverride = null;
QueryPersistence persistenceMode = QueryPersistence.PERSISTENT;
String parameters = null;
boolean trace = false;
// Set expectations
MultivaluedMap<String, String> queryParameters = new MultivaluedMapImpl<>();
queryParameters.putAll(QueryParametersImpl.paramsToMap(queryLogicName, query, queryName, queryVisibility, beginDate, endDate, queryAuthorizations, expirationDate, pagesize, pageTimeout, maxResultsOverride, persistenceMode, parameters, trace));
ColumnVisibilitySecurityMarking marking = new ColumnVisibilitySecurityMarking();
marking.validate(queryParameters);
QueryParameters qp = new QueryParametersImpl();
qp.validate(queryParameters);
MultivaluedMap<String, String> op = new MultivaluedMapImpl<>();
op.putAll(qp.getUnknownParameters(queryParameters));
op.putSingle(PrivateAuditConstants.LOGIC_CLASS, queryLogicName);
op.putSingle(PrivateAuditConstants.COLUMN_VISIBILITY, queryVisibility);
expect(this.queryLogicFactory.getQueryLogic(queryLogicName, this.principal)).andReturn((QueryLogic) this.queryLogic1);
expect(this.context.getCallerPrincipal()).andReturn(this.principal).anyTimes();
queryLogic1.validate(queryParameters);
expect(this.principal.getName()).andReturn("userName Full");
expect(this.principal.getShortName()).andReturn("userName");
expect(this.principal.getUserDN()).andReturn(SubjectIssuerDNPair.of("userDN"));
expect(this.principal.getDNs()).andReturn(new String[] { "userDN" });
expect(this.principal.getProxyServers()).andReturn(new ArrayList<>(0)).anyTimes();
expect(this.queryLogic1.containsDNWithAccess(Collections.singletonList("userDN"))).andReturn(false);
// Run the test
PowerMock.replayAll();
QueryExecutorBean subject = new QueryExecutorBean();
setInternalState(subject, EJBContext.class, context);
setInternalState(subject, QueryLogicFactory.class, queryLogicFactory);
setInternalState(subject, QueryExpirationConfiguration.class, queryExpirationConf);
setInternalState(subject, SecurityMarking.class, new ColumnVisibilitySecurityMarking());
setInternalState(subject, QueryParameters.class, new QueryParametersImpl());
setInternalState(subject, QueryMetricFactory.class, new QueryMetricFactoryImpl());
Throwable result1 = null;
try {
subject.defineQuery(queryLogicName, queryParameters);
} catch (DatawaveWebApplicationException e) {
result1 = e.getCause();
}
PowerMock.verifyAll();
// Verify results
assertTrue("QueryException expected to have been thrown", result1 instanceof QueryException);
assertEquals("Thrown exception expected to have been due to access denied", "401", ((QueryException) result1).getErrorCode());
assertEquals("Thrown exception expected to detail reason for access denial", "None of the DNs used have access to this query logic: [userDN]", result1.getMessage());
}
use of datawave.webservice.query.exception.QueryException in project datawave by NationalSecurityAgency.
the class ExtendedQueryExecutorBeanTest method testCreateQueryAndNext_PageSizeExceedsConfiguredMax.
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testCreateQueryAndNext_PageSizeExceedsConfiguredMax() throws Exception {
// Set local test input
String queryLogicName = "queryLogicName";
String query = "query";
String queryName = "queryName";
String queryVisibility = "A&B";
long currentTime = System.currentTimeMillis();
Date beginDate = new Date(currentTime - 5000);
Date endDate = new Date(currentTime - 1000);
String queryAuthorizations = "AUTH_1";
Date expirationDate = new Date(currentTime + 9999);
int pagesize = 1000;
int pageTimeout = -1;
Long maxResultsOverride = null;
QueryPersistence persistenceMode = QueryPersistence.PERSISTENT;
String parameters = null;
boolean trace = false;
// Set expectations
MultivaluedMap<String, String> queryParameters = new MultivaluedMapImpl<>();
queryParameters.putAll(QueryParametersImpl.paramsToMap(queryLogicName, query, queryName, queryVisibility, beginDate, endDate, queryAuthorizations, expirationDate, pagesize, pageTimeout, maxResultsOverride, persistenceMode, parameters, trace));
ColumnVisibilitySecurityMarking marking = new ColumnVisibilitySecurityMarking();
marking.validate(queryParameters);
QueryParameters qp = new QueryParametersImpl();
qp.validate(queryParameters);
MultivaluedMap<String, String> op = new MultivaluedMapImpl<>();
op.putAll(qp.getUnknownParameters(queryParameters));
op.putSingle(PrivateAuditConstants.LOGIC_CLASS, queryLogicName);
op.putSingle(PrivateAuditConstants.COLUMN_VISIBILITY, queryVisibility);
expect(this.queryLogicFactory.getQueryLogic(queryLogicName, this.principal)).andReturn((QueryLogic) this.queryLogic1);
expect(this.context.getCallerPrincipal()).andReturn(this.principal).anyTimes();
queryLogic1.validate(queryParameters);
expect(this.principal.getName()).andReturn("userName Full");
expect(this.principal.getShortName()).andReturn("userName");
expect(this.principal.getUserDN()).andReturn(SubjectIssuerDNPair.of("userDN"));
expect(this.principal.getDNs()).andReturn(new String[] { "userDN" });
expect(this.principal.getProxyServers()).andReturn(new ArrayList<>(0)).anyTimes();
expect(this.queryLogic1.containsDNWithAccess(Collections.singletonList("userDN"))).andReturn(true);
expect(this.principal.getAuthorizations()).andReturn((Collection) Arrays.asList(Arrays.asList(queryAuthorizations)));
expect(this.queryLogic1.getMaxPageSize()).andReturn(10).times(4);
// Run the test
PowerMock.replayAll();
QueryExecutorBean subject = new QueryExecutorBean();
setInternalState(subject, EJBContext.class, context);
setInternalState(subject, QueryLogicFactory.class, queryLogicFactory);
setInternalState(subject, QueryExpirationConfiguration.class, queryExpirationConf);
setInternalState(subject, SecurityMarking.class, new ColumnVisibilitySecurityMarking());
setInternalState(subject, QueryParameters.class, new QueryParametersImpl());
setInternalState(subject, QueryMetricFactory.class, new QueryMetricFactoryImpl());
Throwable result1 = null;
try {
subject.createQueryAndNext(queryLogicName, queryParameters);
} catch (DatawaveWebApplicationException e) {
result1 = e.getCause();
}
PowerMock.verifyAll();
// Verify results
assertTrue("QueryException expected to have been thrown", result1 instanceof QueryException);
assertEquals("Thrown exception expected to have been due to undefined query logic", "400-6", ((QueryException) result1).getErrorCode());
}
use of datawave.webservice.query.exception.QueryException in project datawave by NationalSecurityAgency.
the class DefaultQueryPlanner method configureTypeMappings.
public static void configureTypeMappings(ShardQueryConfiguration config, IteratorSetting cfg, MetadataHelper metadataHelper, boolean compressMappings) throws DatawaveQueryException {
try {
addOption(cfg, QueryOptions.QUERY_MAPPING_COMPRESS, Boolean.valueOf(compressMappings).toString(), false);
// now lets filter the query field datatypes to those that are not
// indexed
Multimap<String, Type<?>> nonIndexedQueryFieldsDatatypes = HashMultimap.create(config.getQueryFieldsDatatypes());
nonIndexedQueryFieldsDatatypes.keySet().removeAll(config.getIndexedFields());
String nonIndexedTypes = QueryOptions.buildFieldNormalizerString(nonIndexedQueryFieldsDatatypes);
String typeMetadataString = metadataHelper.getTypeMetadata(config.getDatatypeFilter()).toString();
String requiredAuthsString = metadataHelper.getUsersMetadataAuthorizationSubset();
if (compressMappings) {
nonIndexedTypes = QueryOptions.compressOption(nonIndexedTypes, QueryOptions.UTF8);
typeMetadataString = QueryOptions.compressOption(typeMetadataString, QueryOptions.UTF8);
requiredAuthsString = QueryOptions.compressOption(requiredAuthsString, QueryOptions.UTF8);
}
addOption(cfg, QueryOptions.NON_INDEXED_DATATYPES, nonIndexedTypes, false);
addOption(cfg, QueryOptions.TYPE_METADATA, typeMetadataString, false);
addOption(cfg, QueryOptions.TYPE_METADATA_AUTHS, requiredAuthsString, false);
addOption(cfg, QueryOptions.METADATA_TABLE_NAME, config.getMetadataTableName(), false);
} catch (TableNotFoundException | IOException e) {
QueryException qe = new QueryException(DatawaveErrorCode.TYPE_MAPPING_CONFIG_ERROR, e);
throw new DatawaveQueryException(qe);
}
}
use of datawave.webservice.query.exception.QueryException in project datawave by NationalSecurityAgency.
the class DefaultQueryPlanner method processTree.
protected ASTJexlScript processTree(final ASTJexlScript originalQueryTree, ShardQueryConfiguration config, Query settings, MetadataHelper metadataHelper, ScannerFactory scannerFactory, QueryData queryData, QueryStopwatch timers, QueryModel queryModel) throws DatawaveQueryException {
config.setQueryTree(originalQueryTree);
TraceStopwatch stopwatch = null;
if (!disableWhindexFieldMappings) {
// apply the value-specific field mappings for GeoWave functions
config.setQueryTree(timedApplyWhindexFieldMappings(timers, config.getQueryTree(), config, metadataHelper, settings));
}
if (!disableExpandIndexFunction) {
// expand the index queries for the functions
config.setQueryTree(timedExpandIndexQueriesForFunctions(timers, config.getQueryTree(), config, metadataHelper));
}
// apply the node transform rules
// running it here before any unfielded expansions to enable potentially pushing down terms before index lookups
config.setQueryTree(timedApplyNodeTransformRules(timers, "Apply Node Transform Rules - Pre Unfielded Expansions", config.getQueryTree(), config, metadataHelper, getTransformRules()));
// left as a regex
if (!disableAnyFieldLookup) {
config.setQueryTree(timedExpandAnyFieldRegexNodes(timers, config.getQueryTree(), config, metadataHelper, scannerFactory, settings.getQuery()));
}
if (reduceQuery) {
config.setQueryTree(timedReduce(timers, "Reduce Query After ANYFIELD Expansions", config.getQueryTree()));
}
if (!disableTestNonExistentFields) {
timedTestForNonExistentFields(timers, config.getQueryTree(), config, metadataHelper, queryModel, settings);
}
// apply the node transform rules
// running it here before any regex or range expansions to enable potentially pushing down terms before index lookups
config.setQueryTree(timedApplyNodeTransformRules(timers, "Apply Node Transform Rules - Pre Regex/Range Expansions", config.getQueryTree(), config, metadataHelper, getTransformRules()));
timedFetchDatatypes(timers, "Fetch Required Datatypes", config.getQueryTree(), config);
config.setQueryTree(timedFixUnindexedNumerics(timers, config.getQueryTree(), config));
config.setQueryTree(timedExpandMultiNormalizedTerms(timers, config.getQueryTree(), config, metadataHelper));
// if we have any index holes, then mark em
if (!config.getIndexHoles().isEmpty()) {
config.setQueryTree(timedMarkIndexHoles(timers, config.getQueryTree(), config, metadataHelper));
}
// lets precompute the indexed fields and index only fields for the specific datatype if needed below
Set<String> indexedFields = null;
Set<String> indexOnlyFields = null;
Set<String> nonEventFields = null;
if (config.getMinSelectivity() > 0 || !disableBoundedLookup) {
try {
indexedFields = metadataHelper.getIndexedFields(config.getDatatypeFilter());
indexOnlyFields = metadataHelper.getIndexOnlyFields(config.getDatatypeFilter());
nonEventFields = metadataHelper.getNonEventFields(config.getDatatypeFilter());
} catch (TableNotFoundException te) {
QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, te);
throw new DatawaveFatalQueryException(qe);
}
}
// apply the node transform rules
config.setQueryTree(timedApplyNodeTransformRules(timers, "Apply Node Transform Rules - Pre Pushdown/Pullup Expansions", config.getQueryTree(), config, metadataHelper, getTransformRules()));
// push down terms that are over the min selectivity
if (config.getMinSelectivity() > 0) {
config.setQueryTree(timedPushdownLowSelectiveTerms(timers, config.getQueryTree(), config, indexedFields, indexOnlyFields, nonEventFields));
}
config.setQueryTree(timedForceFieldToFieldComparison(timers, config.getQueryTree()));
if (!disableCompositeFields) {
config.setQueryTree(timedExpandCompositeFields(timers, config.getQueryTree(), config));
}
if (!disableBoundedLookup) {
stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Expand bounded query ranges (total)");
// Expand any bounded ranges into a conjunction of discrete terms
try {
Map<String, IndexLookup> indexLookupMap = new HashMap<>();
// Check if there is any regex to expand.
NodeTypeCount nodeCount = NodeTypeCountVisitor.countNodes(config.getQueryTree());
if (nodeCount.hasAny(ASTNRNode.class, ASTERNode.class)) {
config.setQueryTree(timedExpandRegex(timers, "Expand Regex", config.getQueryTree(), config, metadataHelper, scannerFactory, indexLookupMap));
}
// Check if there are any bounded ranges to expand.
if (nodeCount.isPresent(BoundedRange.class)) {
config.setQueryTree(timedExpandRanges(timers, "Expand Ranges", config.getQueryTree(), config, metadataHelper, scannerFactory));
}
// NOTE: GeoWavePruningVisitor should run before QueryPruningVisitor. If it runs after, there is a chance
// that GeoWavePruningVisitor will prune all of the remaining indexed terms, which would leave a GeoWave
// function without any indexed terms or ranges, which should evaluate to false. That case won't be handled
// properly if we run GeoWavePruningVisitor after QueryPruningVisitor.
config.setQueryTree(timedPruneGeoWaveTerms(timers, config.getQueryTree(), metadataHelper));
if (reduceQuery) {
config.setQueryTree(timedReduce(timers, "Reduce Query After Range Expansion", config.getQueryTree()));
}
// Check if there are functions that can be pushed into exceeded value ranges.
if (nodeCount.hasAll(ASTFunctionNode.class, ExceededValueThresholdMarkerJexlNode.class)) {
config.setQueryTree(timedPushFunctions(timers, config.getQueryTree(), config, metadataHelper));
}
if (executableExpansion) {
config.setQueryTree(timedExecutableExpansion(timers, config.getQueryTree(), config, metadataHelper));
}
List<String> debugOutput = null;
if (log.isDebugEnabled()) {
debugOutput = new ArrayList<>(32);
}
// the terms to be delayed.
if (!ExecutableDeterminationVisitor.isExecutable(config.getQueryTree(), config, indexedFields, indexOnlyFields, nonEventFields, debugOutput, metadataHelper)) {
// if we now have an unexecutable tree because of delayed
// predicates, then remove delayed predicates as needed and
// reexpand
config.setQueryTree(timedRemoveDelayedPredicates(timers, "Remove Delayed Predicates", config.getQueryTree(), config, metadataHelper, indexedFields, indexOnlyFields, nonEventFields, indexLookupMap, scannerFactory, metadataHelper, debugOutput));
}
// if we now have an unexecutable tree because of missing
// delayed predicates, then add delayed predicates where
// possible
config.setQueryTree(timedAddDelayedPredicates(timers, "Add Delayed Predicates", config.getQueryTree(), config, metadataHelper, indexedFields, indexOnlyFields, nonEventFields, debugOutput));
} catch (TableNotFoundException e) {
stopwatch.stop();
QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, e);
throw new DatawaveFatalQueryException(qe);
} catch (CannotExpandUnfieldedTermFatalException e) {
if (null != e.getCause() && e.getCause() instanceof DoNotPerformOptimizedQueryException)
throw (DoNotPerformOptimizedQueryException) e.getCause();
QueryException qe = new QueryException(DatawaveErrorCode.INDETERMINATE_INDEX_STATUS, e);
throw new DatawaveFatalQueryException(qe);
}
stopwatch.stop();
} else {
if (log.isDebugEnabled()) {
log.debug("Bounded range and regex conversion has been disabled");
}
}
return config.getQueryTree();
}
use of datawave.webservice.query.exception.QueryException in project datawave by NationalSecurityAgency.
the class DefaultQueryPlanner method timedExpandCompositeFields.
protected ASTJexlScript timedExpandCompositeFields(QueryStopwatch timers, ASTJexlScript script, ShardQueryConfiguration config) throws DatawaveQueryException {
try {
config.setCompositeToFieldMap(metadataHelper.getCompositeToFieldMap(config.getDatatypeFilter()));
config.setCompositeTransitionDates(metadataHelper.getCompositeTransitionDateMap(config.getDatatypeFilter()));
config.setCompositeFieldSeparators(metadataHelper.getCompositeFieldSeparatorMap(config.getDatatypeFilter()));
config.setFieldToDiscreteIndexTypes(CompositeUtils.getFieldToDiscreteIndexTypeMap(config.getQueryFieldsDatatypes()));
} catch (TableNotFoundException e) {
QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, e);
throw new DatawaveFatalQueryException(qe);
}
return visitorManager.timedVisit(timers, "Expand Composite Terms", () -> (ExpandCompositeTerms.expandTerms(config, script)));
}
Aggregations