Examples with MetadataHelper datawave.query.util.MetadataHelper used on opensource projects

Example 1 with MetadataHelper

use of datawave.query.util.MetadataHelper in project datawave by NationalSecurityAgency.

the class DefaultExtendedEdgeQueryLogic method setupQuery.

@Override
public void setupQuery(GenericQueryConfiguration configuration) throws Exception {
    EdgeExtendedSummaryConfiguration localConf = (EdgeExtendedSummaryConfiguration) configuration;
    config = (EdgeExtendedSummaryConfiguration) configuration;
    prefilterValues = null;
    EdgeExtendedSummaryConfiguration.dateType dateFilterType = localConf.getDateRangeType();
    if (log.isTraceEnabled()) {
        log.trace("Performing edge table query: " + config.getQueryString());
    }
    // TODO check to see if overriding I/O necessary
    if (allowOverrideIO && localConf.isOverRideInput()) {
        this.summaryInputType = localConf.isSummaryInputType();
    }
    if (allowOverrideIO && localConf.isOverRideOutput()) {
        this.summaryOutputType = localConf.isAggregateResults();
    }
    boolean includeStats = localConf.includeStats();
    String queryString = config.getQueryString();
    MetadataHelper metadataHelper = super.prepareMetadataHelper(config.getConnector(), config.getModelTableName(), config.getAuthorizations());
    loadQueryModel(metadataHelper, config);
    // list of source values...no field names to translate
    if (this.summaryInputType == false) {
        queryString = applyQueryModel(queryString);
    }
    // set the modified queryString back into the config, for easy access
    config.setQueryString(queryString);
    String normalizedQuery = "";
    String statsNormalizedQuery = "";
    QueryData qData = configureRanges(queryString);
    setRanges(qData.getRanges());
    VisitationContext context = null;
    if (this.summaryInputType == false) {
        try {
            context = normalizeJexlQuery(queryString, false);
            normalizedQuery = context.getNormalizedQuery().toString();
            statsNormalizedQuery = context.getNormalizedStatsQuery().toString();
            if (log.isTraceEnabled()) {
                log.trace("Jexl after normalizing both vertices: " + normalizedQuery);
            }
        } catch (JexlException ex) {
            try {
                log.error("Error parsing user query.", ex);
            } catch (Exception ex2) {
                log.error("Exception thrown by logger (???)");
            }
        }
    }
    if ((null == normalizedQuery || normalizedQuery.equals("")) && qData.getRanges().size() < 1) {
        throw new IllegalStateException("Query string is empty after initial processing, no ranges or filters can be generated to execute.");
    }
    addIterators(qData, getDateBasedIterators(config.getBeginDate(), config.getEndDate(), currentIteratorPriority, dateFilterSkipLimit, dateFilterScanLimit, dateFilterType));
    if (!normalizedQuery.equals("")) {
        if (log.isTraceEnabled()) {
            log.trace("Query being sent to the filter iterator: " + normalizedQuery);
        }
        IteratorSetting edgeIteratorSetting = new IteratorSetting(currentIteratorPriority, EdgeFilterIterator.class.getSimpleName() + "_" + currentIteratorPriority, EdgeFilterIterator.class);
        edgeIteratorSetting.addOption(EdgeFilterIterator.JEXL_OPTION, normalizedQuery);
        edgeIteratorSetting.addOption(EdgeFilterIterator.PROTOBUF_OPTION, "TRUE");
        if (!statsNormalizedQuery.equals("")) {
            edgeIteratorSetting.addOption(EdgeFilterIterator.JEXL_STATS_OPTION, statsNormalizedQuery);
        }
        if (prefilterValues != null) {
            String value = serializePrefilter();
            edgeIteratorSetting.addOption(EdgeFilterIterator.PREFILTER_WHITELIST, value);
        }
        if (includeStats) {
            edgeIteratorSetting.addOption(EdgeFilterIterator.INCLUDE_STATS_OPTION, "TRUE");
        } else {
            edgeIteratorSetting.addOption(EdgeFilterIterator.INCLUDE_STATS_OPTION, "FALSE");
        }
        addIterator(qData, edgeIteratorSetting);
    }
    if (log.isTraceEnabled()) {
        log.trace("Configuring connection: tableName: " + config.getTableName() + ", auths: " + config.getAuthorizations());
    }
    BatchScanner scanner = createBatchScanner(config);
    if (log.isTraceEnabled()) {
        log.trace("Using the following ranges: " + qData.getRanges());
    }
    if (context != null && context.isHasAllCompleteColumnFamilies()) {
        for (Text columnFamily : context.getColumnFamilies()) {
            scanner.fetchColumnFamily(columnFamily);
        }
    }
    scanner.setRanges(qData.getRanges());
    addCustomFilters(qData, currentIteratorPriority);
    for (IteratorSetting setting : qData.getSettings()) {
        scanner.addScanIterator(setting);
    }
    this.scanner = scanner;
    iterator = scanner.iterator();
}

Example 2 with MetadataHelper

use of datawave.query.util.MetadataHelper in project datawave by NationalSecurityAgency.

the class MutableMetadataHandler method process.

public void process(Connector con, ModificationRequestBase request, Map<String, Set<String>> mutableFieldList, Set<Authorizations> userAuths, String user, boolean purgeIndex, boolean insertHistory) throws Exception {
    DefaultModificationRequest mr = DefaultModificationRequest.class.cast(request);
    if (null == mr.getEvents() || mr.getEvents().isEmpty()) {
        throw new IllegalArgumentException("No events specified for modification");
    }
    String fieldName = mr.getFieldName();
    MetadataHelper helper = getMetadataHelper(con);
    MODE mode = mr.getMode();
    MultiTableBatchWriter writer = con.createMultiTableBatchWriter(new BatchWriterConfig().setMaxLatency(1, TimeUnit.SECONDS).setMaxMemory(1048576L).setMaxWriteThreads(4));
    try {
        for (EventIdentifier e : mr.getEvents()) {
            String shardId = e.getShardId();
            String datatype = e.getDatatype();
            Set<String> datatypeFilter = Collections.singleton(datatype);
            String eventUid = e.getEventUid();
            String oldFieldValue = null;
            Map<String, String> oldFieldMarkings = null;
            String oldColumnVisibility = null;
            List<Pair<Key, Value>> currentEntryList = null;
            int valHistoryCount = 0;
            /*
                 * Makes all fields mutable for services requiring no history.
                 */
            if (insertHistory && !isFieldMutable(mutableFieldList, datatype, fieldName))
                throw new IllegalArgumentException("Field " + fieldName + " is not mutable");
            boolean isIndexed = helper.isIndexed(fieldName, datatypeFilter);
            boolean isReverseIndexed = helper.isReverseIndexed(fieldName, datatypeFilter);
            boolean isIndexOnly = helper.getIndexOnlyFields(datatypeFilter).contains(fieldName);
            boolean isContent = (contentFields != null && contentFields.contains(fieldName));
            Set<Type<?>> dataTypes = helper.getDatatypesForField(fieldName, Collections.singleton(datatype));
            if ((isIndexed || isReverseIndexed || isIndexOnly) && (null == dataTypes || dataTypes.isEmpty()))
                throw new IllegalStateException("Field " + fieldName + " is marked index only but has no dataTypes");
            long origTimestamp = getOriginalEventTimestamp(con, userAuths, shardId, datatype, eventUid);
            // Count the history entries if history is going to be inserted.
            if (insertHistory && (MODE.INSERT.equals(mode) || MODE.UPDATE.equals(mode))) {
                List<Pair<Key, Value>> fieldHistoryList = getField(con, userAuths, shardId, datatype, eventUid, "HISTORY_" + fieldName, null, new HashMap<>(), null);
                for (Pair<Key, Value> p : fieldHistoryList) {
                    if (p.getFirst().getColumnQualifier().find(mr.getFieldValue()) > -1) {
                        ++valHistoryCount;
                    }
                }
            }
            if (MODE.UPDATE.equals(mode) || MODE.DELETE.equals(mode)) {
                if (MODE.UPDATE.equals(mode)) {
                    oldFieldValue = mr.getOldFieldValue();
                    oldFieldMarkings = mr.getOldFieldMarkings();
                    oldColumnVisibility = mr.getOldColumnVisibility();
                    if (null == oldFieldValue)
                        throw new IllegalArgumentException("fieldValue parameter required for update");
                } else {
                    oldFieldValue = mr.getFieldValue();
                    oldFieldMarkings = mr.getFieldMarkings();
                    oldColumnVisibility = mr.getColumnVisibility();
                    if (null == oldFieldValue)
                        throw new IllegalArgumentException("fieldValue parameter required for delete");
                }
                ColumnVisibility oldViz = null;
                if (null != oldColumnVisibility) {
                    oldViz = new ColumnVisibility(oldColumnVisibility);
                }
                // find the current values
                currentEntryList = getField(con, userAuths, shardId, datatype, eventUid, fieldName, oldFieldValue, oldFieldMarkings, oldViz);
                if (oldFieldValue != null && currentEntryList.isEmpty()) {
                    throw new IllegalArgumentException("Modification request rejected. Current value of " + fieldName + " does not match submitted value.");
                }
            } else {
                if (null == mr.getFieldValue())
                    throw new IllegalArgumentException("fieldValue parameter required for insert");
            }
            if (MODE.INSERT.equals(mode)) {
                String fieldValue = mr.getFieldValue();
                Map<String, String> fieldMarkings = mr.getFieldMarkings();
                String columnVisibility = mr.getColumnVisibility();
                ColumnVisibility colviz = null;
                if (null != columnVisibility) {
                    colviz = new ColumnVisibility(columnVisibility);
                }
                insert(writer, shardId, datatype, eventUid, fieldMarkings, colviz, fieldName, fieldValue, isIndexOnly, isIndexed, isReverseIndexed, dataTypes, user, MODE.INSERT, origTimestamp + valHistoryCount, insertHistory);
            } else if (MODE.DELETE.equals(mode)) {
                delete(writer, con, userAuths, currentEntryList, isIndexOnly, isIndexed, isReverseIndexed, isContent, dataTypes, user, MODE.DELETE, origTimestamp + valHistoryCount, purgeIndex, insertHistory);
            } else {
                delete(writer, con, userAuths, currentEntryList, isIndexOnly, isIndexed, isReverseIndexed, isContent, dataTypes, user, MODE.UPDATE, origTimestamp + valHistoryCount, purgeIndex, insertHistory);
                String fieldValue = mr.getFieldValue();
                Map<String, String> fieldMarkings = mr.getFieldMarkings();
                String columnVisibility = mr.getColumnVisibility();
                ColumnVisibility colviz = null;
                if (null != columnVisibility) {
                    colviz = new ColumnVisibility(columnVisibility);
                }
                insert(writer, shardId, datatype, eventUid, fieldMarkings, colviz, fieldName, fieldValue, isIndexOnly, isIndexed, isReverseIndexed, dataTypes, user, MODE.UPDATE, origTimestamp + valHistoryCount, insertHistory);
            }
        }
    } finally {
        writer.close();
    }
}

Example 3 with MetadataHelper

use of datawave.query.util.MetadataHelper in project datawave by NationalSecurityAgency.

the class MutableMetadataUUIDHandler method process.

@Override
public void process(Connector con, ModificationRequestBase request, Map<String, Set<String>> mutableFieldList, Set<Authorizations> userAuths, String user) throws BadRequestException, AccumuloException, AccumuloSecurityException, TableNotFoundException, ExecutionException {
    VoidResponse response = new VoidResponse();
    ArrayList<Exception> exceptions = new ArrayList<>();
    MetadataHelper mHelper = getMetadataHelper(con);
    // Receive DefaultUUIDModificationRequest
    DefaultUUIDModificationRequest uuidModReq = DefaultUUIDModificationRequest.class.cast(request);
    List<ModificationEvent> events = uuidModReq.getEvents();
    for (ModificationEvent event : events) {
        List<ModificationOperationImpl> operations = event.getOperations();
        for (ModificationOperationImpl operation : operations) {
            ResetValues();
            OPERATIONMODE mode = operation.getOperationMode();
            String columnVisibility = operation.getColumnVisibility();
            String oldColumnVisibility = operation.getOldColumnVisibility();
            String eventUser = event.getUser();
            // check whether this is a security-marking exempt field. Meaning we can pull the marking if not specified
            boolean securityMarkingExempt = false;
            fieldName = operation.getFieldName();
            for (String s : this.getSecurityMarkingExemptFields()) {
                if (fieldName.toUpperCase().equals(s)) {
                    securityMarkingExempt = true;
                }
            }
            // if they are updating, assume the old values should be the same as current
            if (OPERATIONMODE.UPDATE.equals(mode) && oldColumnVisibility == null) {
                oldColumnVisibility = columnVisibility;
            }
            try {
                if (mHelper.getIndexOnlyFields(mutableFieldList.keySet()).contains(event.getIdType().toUpperCase())) {
                    throw new IllegalStateException("Cannot perform modification because " + event.getIdType() + " is index only. Please search " + "with a different uuidType to identify the event you wish to modify.");
                }
                // perform the lookupUUID
                EventBase<?, ? extends FieldBase<?>> idEvent = findMatchingEventUuid(event.getId(), event.getIdType(), userAuths, operation);
                // extract contents from lookupUUID necessary for modification
                List<? extends FieldBase<?>> fields = idEvent.getFields();
                if (operation.getOldFieldValue() != null)
                    oldFieldValue = operation.getOldFieldValue();
                if (fields != null) {
                    // there may be multiple values for a single field
                    for (FieldBase<?> f : fields) {
                        if (f.getName().equals(fieldName)) {
                            fieldCount++;
                            // if they are doing a replace, we need all the current values, store them
                            if (operation.getOperationMode().equals(OPERATIONMODE.REPLACE)) {
                                if (log != null)
                                    log.trace("Adding " + f.getValueString() + ",delete to replaceMap");
                                replaceMap.put(f.getValueString(), OPERATIONMODE.DELETE);
                            }
                            // user sent an oldValue and we found that value or no oldValue
                            if ((oldFieldValue != null && f.getValueString().equals(oldFieldValue)) || oldFieldValue == null) {
                                fieldValue = f.getValueString();
                                if (columnVisibility == null && securityMarkingExempt) {
                                    fieldColumnVisibility = f.getColumnVisibility();
                                }
                            }
                        } else // only if the input didn't supply a security marking AND it is an exempt field
                        if (f.getName().equalsIgnoreCase(event.getIdType()) && fieldCount < 1 && columnVisibility == null && securityMarkingExempt) {
                            if (log != null)
                                log.trace("Using visibility of " + f.getName() + " and setting to " + f.getColumnVisibility());
                            fieldColumnVisibility = f.getColumnVisibility();
                        }
                    }
                    List<DefaultModificationRequest> modificationRequests = new ArrayList<>();
                    if (OPERATIONMODE.INSERT.equals(mode) || OPERATIONMODE.UPDATE.equals(mode) || OPERATIONMODE.DELETE.equals(mode)) {
                        modificationRequests.add(createModificationRequest(idEvent, operation, columnVisibility, oldColumnVisibility, securityMarkingExempt));
                    } else if (OPERATIONMODE.REPLACE.equals(mode)) {
                        if (log != null)
                            log.trace("Adding " + operation.getFieldValue() + ",insert to replaceMap");
                        replaceMap.put(operation.getFieldValue(), OPERATIONMODE.INSERT);
                        // create a modification request of delete for each current value and an insert for the new value
                        for (String s : replaceMap.keySet()) {
                            ModificationOperation replaceOperation = operation.clone();
                            replaceOperation.setOperationMode(replaceMap.get(s));
                            replaceOperation.setFieldValue(s);
                            oldFieldValue = s;
                            modificationRequests.add(createModificationRequest(idEvent, replaceOperation, columnVisibility, oldColumnVisibility, securityMarkingExempt));
                        }
                    }
                    if (log != null)
                        log.trace("modificationRequests= " + modificationRequests);
                    for (DefaultModificationRequest modReq : modificationRequests) {
                        try {
                            if (fieldCount > 1 && (oldFieldValue == null && modReq.getMode() != MODE.INSERT) && !mode.equals(OPERATIONMODE.REPLACE)) {
                                throw new IllegalStateException("Unable to perform modification. More than one value exists for " + modReq.getFieldName() + ". Please specify the current value you wish to modify in the oldFieldValue field.");
                            } else if (fieldCount < 1 && modReq.getMode() != MODE.INSERT) {
                                throw new IllegalStateException("Unable to perform modification. No values exist for " + modReq.getFieldName() + ".");
                            } else if (columnVisibility == null && !securityMarkingExempt) {
                                throw new IllegalStateException("Must provide columnVisibility");
                            } else // submit DefaultModificationRequest
                            {
                                log.info("eventUser = " + eventUser + ", event.getUser() = " + event.getUser());
                                if (log != null)
                                    log.trace("Submitting request to MutableMetadataHandler from MutableMetadataUUIDHandler: " + modReq);
                                // make sure user isn't null or empty
                                if (eventUser == null || eventUser.equals("")) {
                                    if (log != null)
                                        log.trace("No user provided for event. Using caller: " + user);
                                    super.process(con, modReq, mutableFieldList, userAuths, user);
                                } else {
                                    super.process(con, modReq, mutableFieldList, userAuths, event.getUser());
                                }
                            }
                        }// log exceptions that occur for each modification request. Let as many requests work as possible before returning
                         catch (Exception e) {
                            if (log != null)
                                log.error("Modification error", e);
                            exceptions.add(new Exception(event.getId() + ": " + e.getMessage() + "\n" + modReq));
                        }
                    }
                    modificationRequests.clear();
                } else {
                    throw new IllegalStateException("No event matched " + event.getId());
                }
            } catch (Exception e) {
                if (log != null)
                    log.error("Modification error", e);
                exceptions.add(new Exception(event.getId() + ": " + e.getMessage() + "\n"));
            }
        }
    }
    // If any errors occurred, return them in the response to the user
    if (!exceptions.isEmpty()) {
        for (Exception e : exceptions) {
            QueryException qe = new QueryException(DatawaveErrorCode.MODIFICATION_ERROR, e);
            response.addException(qe.getBottomQueryException());
        }
        QueryException e = new QueryException(DatawaveErrorCode.MODIFICATION_ERROR);
        throw new BadRequestException(e, response);
    }
}

Example 4 with MetadataHelper

use of datawave.query.util.MetadataHelper in project datawave by NationalSecurityAgency.

the class ShardQueryLogic method initialize.

public void initialize(ShardQueryConfiguration config, Connector connection, Query settings, Set<Authorizations> auths) throws Exception {
    // Set the connector and the authorizations into the config object
    config.setConnector(connection);
    config.setAuthorizations(auths);
    config.setMaxScannerBatchSize(getMaxScannerBatchSize());
    config.setMaxIndexBatchSize(getMaxIndexBatchSize());
    setScannerFactory(new ScannerFactory(config));
    // load params before parsing jexl string so these can be injected
    loadQueryParameters(config, settings);
    String jexlQueryString = getJexlQueryString(settings);
    if (null == jexlQueryString) {
        throw new IllegalArgumentException("Query cannot be null");
    } else {
        config.setQueryString(jexlQueryString);
    }
    final Date beginDate = settings.getBeginDate();
    if (null == beginDate) {
        throw new IllegalArgumentException("Begin date cannot be null");
    } else {
        config.setBeginDate(beginDate);
    }
    final Date endDate = settings.getEndDate();
    if (null == endDate) {
        throw new IllegalArgumentException("End date cannot be null");
    } else {
        config.setEndDate(endDate);
    }
    MetadataHelper metadataHelper = prepareMetadataHelper(connection, this.getMetadataTableName(), auths, config.isRawTypes());
    DateIndexHelper dateIndexHelper = prepareDateIndexHelper(connection, this.getDateIndexTableName(), auths);
    if (config.isDateIndexTimeTravel()) {
        dateIndexHelper.setTimeTravel(config.isDateIndexTimeTravel());
    }
    QueryPlanner queryPlanner = getQueryPlanner();
    if (queryPlanner instanceof DefaultQueryPlanner) {
        DefaultQueryPlanner currentQueryPlanner = (DefaultQueryPlanner) queryPlanner;
        currentQueryPlanner.setMetadataHelper(metadataHelper);
        currentQueryPlanner.setDateIndexHelper(dateIndexHelper);
        QueryModelProvider queryModelProvider = currentQueryPlanner.getQueryModelProviderFactory().createQueryModelProvider();
        if (queryModelProvider instanceof MetadataHelperQueryModelProvider) {
            ((MetadataHelperQueryModelProvider) queryModelProvider).setMetadataHelper(metadataHelper);
            ((MetadataHelperQueryModelProvider) queryModelProvider).setConfig(config);
        }
        if (null != queryModelProvider.getQueryModel()) {
            queryModel = queryModelProvider.getQueryModel();
        }
    }
    if (this.queryModel == null)
        loadQueryModel(metadataHelper, config);
    getQueryPlanner().setCreateUidsIteratorClass(createUidsIteratorClass);
    getQueryPlanner().setUidIntersector(uidIntersector);
    validateConfiguration(config);
    if (getCardinalityConfiguration() != null && (!config.getBlacklistedFields().isEmpty() || !config.getProjectFields().isEmpty())) {
        // Ensure that fields used for resultCardinalities are returned. They will be removed in the DocumentTransformer.
        // Modify the projectFields and blacklistFields only for this stage, then return to the original values.
        // Not advisable to create a copy of the config object due to the embedded timers.
        Set<String> originalBlacklistedFields = new HashSet<>(config.getBlacklistedFields());
        Set<String> originalProjectFields = new HashSet<>(config.getProjectFields());
        // this will be caught when loadQueryParameters is called
        if (!config.getBlacklistedFields().isEmpty()) {
            config.setBlacklistedFields(getCardinalityConfiguration().getRevisedBlacklistFields(queryModel, originalBlacklistedFields));
        }
        if (!config.getProjectFields().isEmpty()) {
            config.setProjectFields(getCardinalityConfiguration().getRevisedProjectFields(queryModel, originalProjectFields));
        }
        this.queries = getQueryPlanner().process(config, jexlQueryString, settings, this.getScannerFactory());
        config.setBlacklistedFields(originalBlacklistedFields);
        config.setProjectFields(originalProjectFields);
    } else {
        this.queries = getQueryPlanner().process(config, jexlQueryString, settings, this.getScannerFactory());
    }
    TraceStopwatch stopwatch = config.getTimers().newStartedStopwatch("ShardQueryLogic - Get iterator of queries");
    if (this.queries != null) {
        config.setQueries(this.queries.iterator());
    }
    config.setQueryString(getQueryPlanner().getPlannedScript());
    stopwatch.stop();
}

Example 5 with MetadataHelper

use of datawave.query.util.MetadataHelper in project datawave by NationalSecurityAgency.

the class ShardIndexQueryTable method initialize.

@Override
public GenericQueryConfiguration initialize(Connector connection, Query settings, Set<Authorizations> auths) throws Exception {
    ShardIndexQueryConfiguration config = new ShardIndexQueryConfiguration(this, settings);
    this.scannerFactory = new ScannerFactory(connection);
    MetadataHelper metadataHelper = initializeMetadataHelper(connection, config.getMetadataTableName(), auths);
    if (StringUtils.isEmpty(settings.getQuery())) {
        throw new IllegalArgumentException("Query cannot be null");
    }
    if (log.isDebugEnabled()) {
        log.debug("Query parameters set to " + settings.getParameters());
    }
    String tModelName = getTrimmedOrNull(settings, QueryParameters.PARAMETER_MODEL_NAME);
    if (tModelName != null) {
        modelName = tModelName;
    }
    String tModelTableName = getTrimmedOrNull(settings, QueryParameters.PARAMETER_MODEL_TABLE_NAME);
    if (tModelTableName != null) {
        modelTableName = tModelTableName;
    }
    queryModel = metadataHelper.getQueryModel(modelTableName, modelName, null);
    String datatypeFilterString = getTrimmedOrNull(settings, QueryParameters.DATATYPE_FILTER_SET);
    if (datatypeFilterString != null) {
        config.setDatatypeFilter(new HashSet<>(Arrays.asList(datatypeFilterString.split(PARAM_VALUE_SEP_STR))));
        if (log.isDebugEnabled()) {
            log.debug("Data type filter set to " + config.getDatatypeFilterAsString());
        }
    }
    config.setConnector(connection);
    config.setAuthorizations(auths);
    if (indexTableName != null) {
        config.setIndexTableName(indexTableName);
    }
    if (reverseIndexTableName != null) {
        config.setReverseIndexTableName(reverseIndexTableName);
    }
    if (settings.getBeginDate() != null) {
        config.setBeginDate(settings.getBeginDate());
    } else {
        config.setBeginDate(new Date(0));
        if (log.isDebugEnabled()) {
            log.debug("No begin date supplied in settings.");
        }
    }
    if (settings.getEndDate() != null) {
        config.setEndDate(settings.getEndDate());
    } else {
        config.setEndDate(new Date(Long.MAX_VALUE));
        if (log.isDebugEnabled()) {
            log.debug("No end date supplied in settings.");
        }
    }
    // start with a trimmed version of the query, converted to JEXL
    LuceneToJexlQueryParser parser = new LuceneToJexlQueryParser();
    parser.setAllowLeadingWildCard(this.isAllowLeadingWildcard());
    QueryNode node = parser.parse(settings.getQuery().trim());
    // TODO: Validate that this is a simple list of terms type of query
    config.setQueryString(node.getOriginalQuery());
    if (log.isDebugEnabled()) {
        log.debug("Original Query = " + settings.getQuery().trim());
        log.debug("JEXL Query = " + node.getOriginalQuery());
    }
    // Parse & flatten the query.
    ASTJexlScript origScript = JexlASTHelper.parseAndFlattenJexlQuery(config.getQueryString());
    ASTJexlScript script;
    try {
        script = UnfieldedIndexExpansionVisitor.expandUnfielded(config, this.scannerFactory, metadataHelper, origScript);
    } catch (EmptyUnfieldedTermExpansionException e) {
        Multimap<String, String> emptyMap = Multimaps.unmodifiableMultimap(HashMultimap.create());
        config.setNormalizedTerms(emptyMap);
        config.setNormalizedPatterns(emptyMap);
        return config;
    }
    Set<String> dataTypes = config.getDatatypeFilter();
    Set<String> allFields = metadataHelper.getAllFields(dataTypes);
    script = QueryModelVisitor.applyModel(script, queryModel, allFields);
    if (log.isTraceEnabled()) {
        log.trace("fetching dataTypes from FetchDataTypesVisitor");
    }
    Multimap<String, Type<?>> fieldToDataTypeMap = FetchDataTypesVisitor.fetchDataTypes(metadataHelper, config.getDatatypeFilter(), script);
    config.setDataTypes(fieldToDataTypeMap);
    config.setQueryFieldsDatatypes(fieldToDataTypeMap);
    final Set<String> indexedFields = metadataHelper.getIndexedFields(dataTypes);
    config.setIndexedFields(indexedFields);
    final Set<String> reverseIndexedFields = metadataHelper.getReverseIndexedFields(dataTypes);
    config.setReverseIndexedFields(reverseIndexedFields);
    final Multimap<String, Type<?>> normalizedFields = metadataHelper.getFieldsToDatatypes(dataTypes);
    config.setNormalizedFieldsDatatypes(normalizedFields);
    if (log.isTraceEnabled()) {
        log.trace("Normalizers:");
        for (String field : fieldToDataTypeMap.keySet()) {
            log.trace(field + ": " + fieldToDataTypeMap.get(field));
        }
    }
    script = ExpandMultiNormalizedTerms.expandTerms(config, metadataHelper, script);
    Multimap<String, String> literals = LiteralNodeVisitor.getLiterals(script);
    Multimap<String, String> patterns = PatternNodeVisitor.getPatterns(script);
    Map<Entry<String, String>, Range> rangesForTerms = Maps.newHashMap();
    Map<Entry<String, String>, Entry<Range, Boolean>> rangesForPatterns = Maps.newHashMap();
    config.setNormalizedTerms(literals);
    config.setNormalizedPatterns(patterns);
    if (log.isDebugEnabled()) {
        log.debug("Normalized Literals = " + literals);
        log.debug("Normalized Patterns = " + patterns);
    }
    for (Entry<String, String> entry : literals.entries()) {
        rangesForTerms.put(entry, ShardIndexQueryTableStaticMethods.getLiteralRange(entry));
    }
    for (Entry<String, String> entry : patterns.entries()) {
        ShardIndexQueryTableStaticMethods.RefactoredRangeDescription r = ShardIndexQueryTableStaticMethods.getRegexRange(entry, isFullTableScanEnabled(), metadataHelper, config);
        rangesForPatterns.put(entry, Maps.immutableEntry(r.range, r.isForReverseIndex));
    }
    config.setRangesForTerms(rangesForTerms);
    config.setRangesForPatterns(rangesForPatterns);
    return config;
}