Examples with ShardQueryConfiguration datawave.query.config.ShardQueryConfiguration used on opensource projects

Example 71 with ShardQueryConfiguration

use of datawave.query.config.ShardQueryConfiguration in project datawave by NationalSecurityAgency.

the class FunctionIndexQueryExpansionVisitorTest method expandContentPhraseFunctionIntoSingleFieldWithNoExpansion.

@Test
public void expandContentPhraseFunctionIntoSingleFieldWithNoExpansion() throws ParseException {
    Set<String> fields = Sets.newHashSet("FOO", "BAR");
    Set<String> tfFields = Sets.newHashSet("FOO");
    // Configure the mock metadata helper.
    MockMetadataHelper mockMetadataHelper = new MockMetadataHelper();
    mockMetadataHelper.setIndexedFields(fields);
    mockMetadataHelper.addTermFrequencyFields(tfFields);
    ShardQueryConfiguration config = new ShardQueryConfiguration();
    config.setNoExpansionFields(Sets.newHashSet("FOO"));
    // Execute the test.
    String original = "content:phrase(termOffsetMap, 'abc', 'def')";
    runTest(original, original, config, mockMetadataHelper);
}

Example 72 with ShardQueryConfiguration

use of datawave.query.config.ShardQueryConfiguration in project datawave by NationalSecurityAgency.

the class FieldIndexCountQueryLogic method setupQuery.

/**
 * Create the batch scanner and set the iterator options / stack.
 *
 * @param genericConfig
 *            configuration object
 * @throws Exception
 */
@Override
public void setupQuery(GenericQueryConfiguration genericConfig) throws Exception {
    if (logger.isTraceEnabled()) {
        logger.trace("setupQuery");
    }
    if (!ShardQueryConfiguration.class.isAssignableFrom(genericConfig.getClass())) {
        throw new QueryException("Did not receive a ShardQueryConfiguration instance!!");
    }
    ShardQueryConfiguration config = (ShardQueryConfiguration) genericConfig;
    // Ensure we have all of the information needed to run a query
    if (!config.canRunQuery()) {
        logger.warn("The given query '" + config.getQueryString() + "' could not be run, most likely due to not matching any records in the global index.");
        // Stub out an iterator to correctly present "no results"
        this.iterator = new Iterator<Map.Entry<Key, Value>>() {

            @Override
            public boolean hasNext() {
                return false;
            }

            @Override
            public Map.Entry<Key, Value> next() {
                return null;
            }

            @Override
            public void remove() {
            }
        };
        this.scanner = null;
        return;
    }
    try {
        if (logger.isTraceEnabled()) {
            logger.trace("configuring batch scanner and iterators.");
        }
        BatchScanner bs = getScannerFactory().newScanner(config.getShardTableName(), config.getAuthorizations(), config.getNumQueryThreads(), config.getQuery());
        bs.setRanges(this.ranges);
        // The stack we want to use
        // 21 FieldIndexCountingIterator
        // FieldIndexCountingIterator setup
        IteratorSetting cfg;
        cfg = new IteratorSetting(config.getBaseIteratorPriority() + 21, "countingIter", FieldIndexCountingIterator.class);
        cfg.addOption(FieldIndexCountingIterator.DATA_TYPES, config.getDatatypeFilterAsString());
        cfg.addOption(FieldIndexCountingIterator.FIELD_NAMES, join(this.fieldNames, FieldIndexCountingIterator.SEP));
        if (null != this.fieldValues && !this.fieldValues.isEmpty()) {
            cfg.addOption(FieldIndexCountingIterator.FIELD_VALUES, join(this.fieldValues, FieldIndexCountingIterator.SEP));
        }
        SimpleDateFormat sdf = new SimpleDateFormat(FieldIndexCountingIterator.DATE_FORMAT_STRING);
        cfg.addOption(FieldIndexCountingIterator.START_TIME, sdf.format(config.getBeginDate()));
        cfg.addOption(FieldIndexCountingIterator.STOP_TIME, sdf.format(config.getEndDate()));
        cfg.addOption(FieldIndexCountingIterator.UNIQ_BY_DATA_TYPE, Boolean.toString(this.uniqueByDataType));
        bs.addScanIterator(cfg);
        this.iterator = bs.iterator();
        this.scanner = bs;
    } catch (TableNotFoundException e) {
        logger.error("The table '" + config.getShardTableName() + "' does not exist", e);
    }
}

Example 73 with ShardQueryConfiguration

use of datawave.query.config.ShardQueryConfiguration in project datawave by NationalSecurityAgency.

the class FieldIndexCountQueryLogic method initialize.

@Override
public GenericQueryConfiguration initialize(Connector connection, Query settings, Set<Authorizations> auths) throws Exception {
    if (logger.isTraceEnabled()) {
        logger.trace("initialize");
    }
    this.scannerFactory = new ScannerFactory(connection);
    MetadataHelper metadataHelper = prepareMetadataHelper(connection, this.getMetadataTableName(), auths);
    String modelName = this.getModelName();
    String modelTableName = this.getModelTableName();
    // Check if the default modelName and modelTableNames have been overriden by custom parameters.
    if (null != settings.findParameter(QueryParameters.PARAMETER_MODEL_NAME) && !settings.findParameter(QueryParameters.PARAMETER_MODEL_NAME).getParameterValue().trim().isEmpty()) {
        modelName = settings.findParameter(QueryParameters.PARAMETER_MODEL_NAME).getParameterValue().trim();
    }
    if (null != settings.findParameter(QueryParameters.PARAMETER_MODEL_TABLE_NAME) && !settings.findParameter(QueryParameters.PARAMETER_MODEL_TABLE_NAME).getParameterValue().trim().isEmpty()) {
        modelTableName = settings.findParameter(QueryParameters.PARAMETER_MODEL_TABLE_NAME).getParameterValue().trim();
    }
    if (null != modelName && null == modelTableName) {
        throw new IllegalArgumentException(QueryParameters.PARAMETER_MODEL_NAME + " has been specified but " + QueryParameters.PARAMETER_MODEL_TABLE_NAME + " is missing. Both are required to use a model");
    }
    if (null != modelName && null != modelTableName) {
        this.queryModel = metadataHelper.getQueryModel(modelTableName, modelName, this.getUnevaluatedFields());
    }
    // I'm using this config object in a pinch, we should probably create a custom one.
    ShardQueryConfiguration config = ShardQueryConfiguration.create(this, settings);
    config.setConnector(connection);
    config.setAuthorizations(auths);
    // the following throw IllegalArgumentExceptions if validation fails.
    parseQuery(config, settings);
    configDate(config, settings);
    configTypeFilter(config, settings);
    Set<String> normalizedFieldValues = null;
    Iterator<String> fieldNameIter = fieldNames.iterator();
    while (fieldNameIter.hasNext()) {
        String fieldName = fieldNameIter.next();
        // check that the field name is actually an indexed field
        Set<Type<?>> normalizerSet = metadataHelper.getDatatypesForField(fieldName, config.getDatatypeFilter());
        if (null != normalizerSet && !normalizerSet.isEmpty()) {
            if (null != this.fieldValues && !this.fieldValues.isEmpty()) {
                for (Type<?> norm : normalizerSet) {
                    if (null == normalizedFieldValues) {
                        normalizedFieldValues = new HashSet<>();
                    }
                    for (String val : this.fieldValues) {
                        try {
                            String normVal = norm.normalize(val);
                            if (null != normVal && !normVal.isEmpty()) {
                                normalizedFieldValues.add(normVal);
                            }
                        } catch (Exception e) {
                            logger.debug(norm + " failed to normalize value: " + val);
                        }
                    }
                }
            }
        } else {
            if (logger.isTraceEnabled()) {
                logger.trace("dropping fieldname " + fieldName + " because it's not indexed.");
            }
            // drop fieldName since it isn't indexed.
            fieldNameIter.remove();
        }
    }
    this.fieldValues = normalizedFieldValues;
    if (this.fieldNames.isEmpty()) {
        throw new IllegalArgumentException("Need at least 1 indexed field to query with.");
    }
    // Generate & set the query ranges
    this.ranges = generateRanges(config);
    // Find out if we need to list unique data types.
    if (null != settings.findParameter(Constants.UNIQ_DATATYPE)) {
        this.uniqueByDataType = Boolean.parseBoolean(settings.findParameter(Constants.UNIQ_DATATYPE).getParameterValue());
        if (logger.isTraceEnabled()) {
            logger.trace("uniqueByDataType: " + uniqueByDataType);
        }
    }
    // Find out if we need to list unique visibilities.
    if (null != settings.findParameter(Constants.UNIQ_VISIBILITY)) {
        this.uniqueByVisibility = Boolean.parseBoolean(settings.findParameter(Constants.UNIQ_VISIBILITY).getParameterValue());
        if (logger.isTraceEnabled()) {
            logger.trace("uniqueByVisibility: " + uniqueByVisibility);
        }
    }
    if (logger.isTraceEnabled()) {
        logger.trace("FieldNames: ");
        for (String f : this.fieldNames) {
            logger.trace("\t" + f);
        }
        logger.trace("FieldValues: ");
        if (null == this.fieldValues) {
            logger.trace("\tnone");
        } else {
            for (String f : this.fieldValues) {
                logger.trace("\t" + f);
            }
        }
        logger.trace("uniqueByDataType: " + uniqueByDataType);
        logger.trace("uniqueByVisibility: " + uniqueByVisibility);
    }
    return config;
}

Example 74 with ShardQueryConfiguration

use of datawave.query.config.ShardQueryConfiguration in project datawave by NationalSecurityAgency.

the class IndexStatsQueryLogic method setupQuery.

@Override
public void setupQuery(GenericQueryConfiguration config) throws Exception {
    ShardQueryConfiguration qConf = (ShardQueryConfiguration) config;
    HashSet<String> fields = new HashSet<>();
    Collections.addAll(fields, config.getQueryString().split(" "));
    StatsMonkey monkey = new StatsMonkey();
    monkey.con = connector;
    monkey.table = TableName.INDEX_STATS;
    List<FieldStat> stats = monkey.getStat(fields, qConf.getDatatypeFilter(), qConf.getBeginDate(), qConf.getEndDate());
    this.iterator = stats.iterator();
}

Example 75 with ShardQueryConfiguration

use of datawave.query.config.ShardQueryConfiguration in project datawave by NationalSecurityAgency.

the class IndexStatsQueryLogic method initialize.

@Override
public GenericQueryConfiguration initialize(Connector connector, Query query, Set<Authorizations> auths) throws Exception {
    this.connector = connector;
    ShardQueryConfiguration config = new ShardQueryConfiguration();
    // Get the datatype set if specified
    String typeList = query.findParameter(QueryParameters.DATATYPE_FILTER_SET).getParameterValue();
    HashSet<String> typeFilter = null;
    if (null != typeList && !typeList.isEmpty()) {
        typeFilter = new HashSet<>();
        typeFilter.addAll(Arrays.asList(StringUtils.split(typeList, Constants.PARAM_VALUE_SEP)));
        if (!typeFilter.isEmpty()) {
            config.setDatatypeFilter(typeFilter);
            if (log.isDebugEnabled()) {
                log.debug("Type Filter: " + typeFilter);
            }
        }
    }
    config.setBeginDate(query.getBeginDate());
    config.setEndDate(query.getEndDate());
    config.setQueryString(query.getQuery());
    config.setAuthorizations(auths);
    return config;
}