use of datawave.query.exceptions.DatawaveFatalQueryException in project datawave by NationalSecurityAgency.
the class KeyToDocumentData method apply.
@Override
public Entry<DocumentData, Document> apply(Entry<Key, Document> from) {
// We want to ensure that we have a non-empty colqual
if (null == from || null == from.getKey() || null == from.getValue())
return null;
Range keyRange = getKeyRange(from);
try {
source.seek(keyRange, columnFamilies, inclusive);
if (log.isDebugEnabled())
log.debug(source.hasTop() + " Key range is " + keyRange);
// Assign only once for
final List<Entry<Key, Value>> attrs;
// efficiency
final Set<Key> docKeys = new HashSet<>();
if (source.hasTop()) {
attrs = this.collectDocumentAttributes(from.getKey(), docKeys, keyRange);
this.appendHierarchyFields(attrs, keyRange, from.getKey());
} else {
attrs = Collections.emptyList();
}
return Maps.immutableEntry(new DocumentData(from.getKey(), docKeys, attrs, false), from.getValue());
} catch (IOException e) {
log.error("Unable to collection document attributes for evaluation: " + keyRange, e);
QueryException qe = new QueryException(DatawaveErrorCode.DOCUMENT_EVALUATION_ERROR, e);
throw new DatawaveFatalQueryException(qe);
}
}
use of datawave.query.exceptions.DatawaveFatalQueryException in project datawave by NationalSecurityAgency.
the class IndexOnlyKeyToDocumentData method next.
@Override
public Entry<DocumentData, Document> next() {
final Entry<Key, Value> next;
try {
next = this.seekNext(false);
} catch (IOException e) {
QueryException qe = new QueryException(DatawaveErrorCode.SEEK_NEXT_ELEMENT_ERROR, e);
throw new DatawaveFatalQueryException(qe);
}
final Entry<DocumentData, Document> entry;
if (null != next) {
final List<Entry<Key, Value>> keyValues = new LinkedList<>();
keyValues.add(next);
Key docKey = getDocKey(next.getKey());
final DocumentData documentData = new DocumentData(this.iteratorDocumentKey, Collections.singleton(docKey), keyValues, true);
entry = Maps.immutableEntry(documentData, this.iteratorDocument);
} else if (next == ITERATOR_COMPLETE_KEY) {
QueryException qe = new QueryException(DatawaveErrorCode.FETCH_NEXT_ELEMENT_ERROR, MessageFormat.format("Fieldname: {0}, Range: {1}", this.fieldName, this.parent));
throw (NoSuchElementException) (new NoSuchElementException().initCause(qe));
} else {
entry = null;
}
return entry;
}
use of datawave.query.exceptions.DatawaveFatalQueryException in project datawave by NationalSecurityAgency.
the class VisitorFunction method getEarliestBeginDate.
private Date getEarliestBeginDate(Collection<Range> ranges) {
SimpleDateFormat sdf = new SimpleDateFormat(DateHelper.DATE_FORMAT_STRING_TO_DAY);
Date minDate = null;
try {
for (Range range : ranges) {
String stringDate = range.getStartKey().getRow().toString();
if (stringDate.length() >= DateHelper.DATE_FORMAT_STRING_TO_DAY.length()) {
stringDate = stringDate.substring(0, DateHelper.DATE_FORMAT_STRING_TO_DAY.length());
Date date = sdf.parse(stringDate);
if (minDate == null || date.compareTo(minDate) < 0) {
minDate = date;
}
}
}
} catch (Exception e) {
throw new DatawaveFatalQueryException(e);
}
return minDate;
}
use of datawave.query.exceptions.DatawaveFatalQueryException in project datawave by NationalSecurityAgency.
the class VisitorFunction method apply.
@Override
@Nullable
public ScannerChunk apply(@Nullable ScannerChunk input) {
SessionOptions options = input.getOptions();
ScannerChunk newSettings = new ScannerChunk(null, input.getRanges(), input.getLastKnownLocation());
SessionOptions newOptions = new SessionOptions(options);
for (IteratorSetting setting : options.getIterators()) {
final String query = setting.getOptions().get(QueryOptions.QUERY);
if (null != query) {
IteratorSetting newIteratorSetting = new IteratorSetting(setting.getPriority(), setting.getName(), setting.getIteratorClass());
newIteratorSetting.addOptions(setting.getOptions());
try {
ASTJexlScript script = null;
boolean evaluatedPreviously = previouslyExecutable(query);
boolean madeChange = false;
if (!evaluatedPreviously && config.isCleanupShardsAndDaysQueryHints()) {
script = JexlASTHelper.parseAndFlattenJexlQuery(query);
script = DateIndexCleanupVisitor.cleanup(script);
madeChange = true;
}
String newQuery = evaluatedPreviously ? previouslyExpanded.get(query) : query;
List<String> debug = null;
if (log.isTraceEnabled())
debug = Lists.newArrayList();
if (!config.isDisableWhindexFieldMappings() && !evaluatedPreviously) {
if (null == script)
script = JexlASTHelper.parseAndFlattenJexlQuery(query);
// apply the whindex using the shard date
ASTJexlScript rebuiltScript = WhindexVisitor.apply(script, config, getEarliestBeginDate(newSettings.getRanges()), metadataHelper);
// if the query changed, save it, and mark it as such
if (!TreeEqualityVisitor.isEqual(script, rebuiltScript)) {
log.debug("[" + config.getQuery().getId() + "] The WhindexVisitor updated the query: " + JexlStringBuildingVisitor.buildQuery(script));
script = rebuiltScript;
madeChange = true;
}
}
if (!config.isBypassExecutabilityCheck() || !evaluatedPreviously) {
if (null == script)
script = JexlASTHelper.parseAndFlattenJexlQuery(query);
if (!ExecutableDeterminationVisitor.isExecutable(script, config, indexedFields, indexOnlyFields, nonEventFields, true, debug, this.metadataHelper)) {
if (log.isTraceEnabled()) {
log.trace("Need to pull up non-executable query: " + JexlStringBuildingVisitor.buildQuery(script));
for (String debugStatement : debug) {
log.trace(debugStatement);
}
DefaultQueryPlanner.logQuery(script, "Failing query:");
}
script = (ASTJexlScript) PullupUnexecutableNodesVisitor.pullupDelayedPredicates(script, true, config, indexedFields, indexOnlyFields, nonEventFields, metadataHelper);
madeChange = true;
STATE state = ExecutableDeterminationVisitor.getState(script, config, indexedFields, indexOnlyFields, nonEventFields, true, debug, metadataHelper);
/**
* We could achieve better performance if we live with the small number of queries that error due to the full table scan exception.
*
* Either look at improving PushdownUnexecutableNodesVisitor or avoid the process altogether.
*/
if (state != STATE.EXECUTABLE) {
if (log.isTraceEnabled()) {
log.trace("Need to push down non-executable query: " + JexlStringBuildingVisitor.buildQuery(script));
for (String debugStatement : debug) {
log.trace(debugStatement);
}
}
script = (ASTJexlScript) PushdownUnexecutableNodesVisitor.pushdownPredicates(script, true, config, indexedFields, indexOnlyFields, nonEventFields, metadataHelper);
}
state = ExecutableDeterminationVisitor.getState(script, config, indexedFields, indexOnlyFields, nonEventFields, true, debug, metadataHelper);
if (state != STATE.EXECUTABLE) {
if (state == STATE.ERROR) {
log.warn("After expanding the query, it is determined that the query cannot be executed due to index-only fields mixed with expressions that cannot be run against the index.");
BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.INDEX_ONLY_FIELDS_MIXED_INVALID_EXPRESSIONS);
throw new InvalidQueryException(qe);
}
log.warn("After expanding the query, it is determined that the query cannot be executed against the field index and a full table scan is required");
if (!config.getFullTableScanEnabled()) {
if (log.isTraceEnabled()) {
log.trace("Full Table fail of " + JexlStringBuildingVisitor.buildQuery(script));
for (String debugStatement : debug) {
log.trace(debugStatement);
}
DefaultQueryPlanner.logQuery(script, "Failing query:");
}
PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.FULL_TABLE_SCAN_REQUIRED_BUT_DISABLED);
throw new DatawaveFatalQueryException(qe);
}
}
if (log.isTraceEnabled()) {
for (String debugStatement : debug) {
log.trace(debugStatement);
}
DefaultQueryPlanner.logQuery(script, "Query pushing down large fielded lists:");
}
}
}
if (config.getSerializeQueryIterator()) {
serializeQuery(newIteratorSetting);
} else {
if (!evaluatedPreviously) {
// if we have an hdfs configuration, then we can pushdown large fielded lists to an ivarator
if (config.getHdfsSiteConfigURLs() != null && setting.getOptions().get(QueryOptions.BATCHED_QUERY) == null) {
if (null == script)
script = JexlASTHelper.parseAndFlattenJexlQuery(query);
try {
script = pushdownLargeFieldedLists(config, script);
madeChange = true;
} catch (IOException ioe) {
log.error("Unable to pushdown large fielded lists....leaving in expanded form", ioe);
}
}
}
}
// only recompile the script if changes were made to the query
if (madeChange)
newQuery = JexlStringBuildingVisitor.buildQuery(script);
try {
previouslyExpanded.put(query, newQuery);
} catch (NullPointerException npe) {
throw new DatawaveFatalQueryException(String.format("New query is null! madeChange: %b, qid: %s", madeChange, setting.getOptions().get(QueryOptions.QUERY_ID)), npe);
}
// test the final script for thresholds
DefaultQueryPlanner.validateQuerySize("VisitorFunction", script, config, false);
newIteratorSetting.addOption(QueryOptions.QUERY, newQuery);
newOptions.removeScanIterator(setting.getName());
newOptions.addScanIterator(newIteratorSetting);
if (log.isDebugEnabled()) {
log.debug("VisitorFunction result: " + newSettings.getRanges());
}
if (log.isTraceEnabled()) {
DefaultQueryPlanner.logTrace(PrintingVisitor.formattedQueryStringList(script), "VistorFunction::apply method");
} else if (log.isDebugEnabled()) {
DefaultQueryPlanner.logDebug(PrintingVisitor.formattedQueryStringList(script, DefaultQueryPlanner.maxChildNodesToPrint), "VistorFunction::apply method");
}
} catch (ParseException e) {
throw new DatawaveFatalQueryException(e);
}
}
}
newSettings.setOptions(newOptions);
return newSettings;
}
use of datawave.query.exceptions.DatawaveFatalQueryException in project datawave by NationalSecurityAgency.
the class IteratorBuildingVisitor method visit.
@Override
public Object visit(ASTAndNode and, Object data) {
QueryPropertyMarker.Instance instance = QueryPropertyMarker.findInstance(and);
if (instance.isType(ExceededOrThresholdMarkerJexlNode.class)) {
JexlNode source = instance.getSource();
// Ivarator to get the job done
if (source instanceof ASTAndNode) {
try {
ivarateList(and, source, data);
} catch (IOException ioe) {
throw new DatawaveFatalQueryException(ioe);
}
} else {
QueryException qe = new QueryException(DatawaveErrorCode.UNEXPECTED_SOURCE_NODE, MessageFormat.format("{0}", "Limited ExceededOrThresholdMarkerJexlNode"));
throw new DatawaveFatalQueryException(qe);
}
} else if (data instanceof IndexRangeIteratorBuilder) {
// index checking has already been done, otherwise we would not have
// an "ExceededValueThresholdMarker"
// hence the "IndexAgnostic" method can be used here
LiteralRange range = JexlASTHelper.findRange().recursively().getRange(and);
if (range == null) {
QueryException qe = new QueryException(DatawaveErrorCode.MULTIPLE_RANGES_IN_EXPRESSION);
throw new DatawaveFatalQueryException(qe);
}
((IndexRangeIteratorBuilder) data).setRange(range);
} else if (instance.isType(ExceededValueThresholdMarkerJexlNode.class)) {
// if the parent is our ExceededValueThreshold marker, then use an
// Ivarator to get the job done unless we don't have to
JexlNode source = instance.getSource();
String identifier = null;
LiteralRange<?> range = null;
boolean negatedLocal = false;
if (source instanceof ASTAndNode) {
range = buildLiteralRange(source, null);
identifier = range.getFieldName();
} else {
if (source instanceof ASTNRNode || source instanceof ASTNotNode)
negatedLocal = true;
range = buildLiteralRange(source);
identifier = JexlASTHelper.getIdentifier(source);
}
boolean negatedOverall = negatedLocal;
if (data instanceof AbstractIteratorBuilder) {
AbstractIteratorBuilder oib = (AbstractIteratorBuilder) data;
if (oib.isInANot()) {
negatedOverall = !negatedOverall;
}
}
// or the field is index only but not in the term frequencies, then we must ivarate
if (!limitLookup || !allowTermFrequencyLookup || (indexOnlyFields.contains(identifier) && !termFrequencyFields.contains(identifier))) {
if (source instanceof ASTAndNode) {
try {
List<ASTFunctionNode> functionNodes = JexlASTHelper.getFunctionNodes(source).stream().filter(node -> JexlFunctionArgumentDescriptorFactory.F.getArgumentDescriptor(node).allowIvaratorFiltering()).collect(Collectors.toList());
if (functionNodes.isEmpty()) {
ivarateRange(and, source, data);
} else {
ivarateFilter(and, source, data, functionNodes);
}
} catch (IOException ioe) {
throw new DatawaveFatalQueryException("Unable to ivarate", ioe);
}
} else if (source instanceof ASTERNode || source instanceof ASTNRNode) {
try {
ivarateRegex(and, source, data);
} catch (IOException ioe) {
throw new DatawaveFatalQueryException("Unable to ivarate", ioe);
}
} else {
QueryException qe = new QueryException(DatawaveErrorCode.UNEXPECTED_SOURCE_NODE, MessageFormat.format("{0}", "ExceededValueThresholdMarkerJexlNode"));
throw new DatawaveFatalQueryException(qe);
}
} else {
NestedIterator<Key> nested = null;
if (termFrequencyFields.contains(identifier)) {
nested = buildExceededFromTermFrequency(identifier, and, source, range, data);
} else {
/**
* This is okay since 1) We are doc specific 2) We are not index only or tf 3) Therefore, we must evaluate against the document for this
* expression 4) Return a stubbed range in case we have a disjunction that breaks the current doc.
*/
if (!limitOverride && !negatedOverall)
nested = createExceededCheck(identifier, range, and);
}
if (null != nested && null != data && data instanceof AbstractIteratorBuilder) {
AbstractIteratorBuilder iterators = (AbstractIteratorBuilder) data;
if (negatedLocal) {
iterators.addExclude(nested);
} else {
iterators.addInclude(nested);
}
} else {
if (isQueryFullySatisfied == true) {
log.warn("Determined that isQueryFullySatisfied should be false, but it was not preset to false in the SatisfactionVisitor");
}
return nested;
}
}
} else if (null != data && data instanceof AndIteratorBuilder) {
and.childrenAccept(this, data);
} else {
// Create an AndIterator and recursively add the children
AbstractIteratorBuilder andItr = new AndIteratorBuilder();
andItr.negateAsNeeded(data);
and.childrenAccept(this, andItr);
// If there is no parent
if (data == null) {
// Make this AndIterator the root node
if (!andItr.includes().isEmpty()) {
root = andItr.build();
}
} else {
// Otherwise, add this AndIterator to its parent
AbstractIteratorBuilder parent = (AbstractIteratorBuilder) data;
if (!andItr.includes().isEmpty()) {
parent.addInclude(andItr.build());
}
}
if (log.isTraceEnabled()) {
log.trace("ASTAndNode visit: pretty formatting of:\nparent.includes:" + formatIncludesOrExcludes(andItr.includes()) + "\nparent.excludes:" + formatIncludesOrExcludes(andItr.excludes()));
}
}
return null;
}
Aggregations