Examples with ScannerFactory datawave.query.tables.ScannerFactory used on opensource projects

Example 21 with ScannerFactory

use of datawave.query.tables.ScannerFactory in project datawave by NationalSecurityAgency.

the class ContentQueryTable method initialize.

@Override
public GenericQueryConfiguration initialize(final Connector connection, final Query settings, final Set<Authorizations> auths) throws Exception {
    // Initialize the config and scanner factory
    final ContentQueryConfiguration config = new ContentQueryConfiguration(this, settings);
    this.scannerFactory = new ScannerFactory(connection);
    config.setConnector(connection);
    config.setAuthorizations(auths);
    // Re-assign the view name if specified via params
    Parameter p = settings.findParameter(QueryParameters.CONTENT_VIEW_NAME);
    if (null != p && !StringUtils.isEmpty(p.getParameterValue())) {
        this.viewName = p.getParameterValue();
    }
    // Decide whether or not to include the content of child events
    String end;
    p = settings.findParameter(QueryParameters.CONTENT_VIEW_ALL);
    if ((null != p) && (null != p.getParameterValue()) && StringUtils.isNotBlank(p.getParameterValue())) {
        end = ALL;
    } else {
        end = PARENT_ONLY;
    }
    // Configure ranges
    final Collection<Range> ranges = this.createRanges(settings, end);
    config.setRanges(ranges);
    return config;
}

Example 22 with ScannerFactory

use of datawave.query.tables.ScannerFactory in project datawave by NationalSecurityAgency.

the class FieldIndexCountQueryLogic method initialize.

@Override
public GenericQueryConfiguration initialize(Connector connection, Query settings, Set<Authorizations> auths) throws Exception {
    if (logger.isTraceEnabled()) {
        logger.trace("initialize");
    }
    this.scannerFactory = new ScannerFactory(connection);
    MetadataHelper metadataHelper = prepareMetadataHelper(connection, this.getMetadataTableName(), auths);
    String modelName = this.getModelName();
    String modelTableName = this.getModelTableName();
    // Check if the default modelName and modelTableNames have been overriden by custom parameters.
    if (null != settings.findParameter(QueryParameters.PARAMETER_MODEL_NAME) && !settings.findParameter(QueryParameters.PARAMETER_MODEL_NAME).getParameterValue().trim().isEmpty()) {
        modelName = settings.findParameter(QueryParameters.PARAMETER_MODEL_NAME).getParameterValue().trim();
    }
    if (null != settings.findParameter(QueryParameters.PARAMETER_MODEL_TABLE_NAME) && !settings.findParameter(QueryParameters.PARAMETER_MODEL_TABLE_NAME).getParameterValue().trim().isEmpty()) {
        modelTableName = settings.findParameter(QueryParameters.PARAMETER_MODEL_TABLE_NAME).getParameterValue().trim();
    }
    if (null != modelName && null == modelTableName) {
        throw new IllegalArgumentException(QueryParameters.PARAMETER_MODEL_NAME + " has been specified but " + QueryParameters.PARAMETER_MODEL_TABLE_NAME + " is missing. Both are required to use a model");
    }
    if (null != modelName && null != modelTableName) {
        this.queryModel = metadataHelper.getQueryModel(modelTableName, modelName, this.getUnevaluatedFields());
    }
    // I'm using this config object in a pinch, we should probably create a custom one.
    ShardQueryConfiguration config = ShardQueryConfiguration.create(this, settings);
    config.setConnector(connection);
    config.setAuthorizations(auths);
    // the following throw IllegalArgumentExceptions if validation fails.
    parseQuery(config, settings);
    configDate(config, settings);
    configTypeFilter(config, settings);
    Set<String> normalizedFieldValues = null;
    Iterator<String> fieldNameIter = fieldNames.iterator();
    while (fieldNameIter.hasNext()) {
        String fieldName = fieldNameIter.next();
        // check that the field name is actually an indexed field
        Set<Type<?>> normalizerSet = metadataHelper.getDatatypesForField(fieldName, config.getDatatypeFilter());
        if (null != normalizerSet && !normalizerSet.isEmpty()) {
            if (null != this.fieldValues && !this.fieldValues.isEmpty()) {
                for (Type<?> norm : normalizerSet) {
                    if (null == normalizedFieldValues) {
                        normalizedFieldValues = new HashSet<>();
                    }
                    for (String val : this.fieldValues) {
                        try {
                            String normVal = norm.normalize(val);
                            if (null != normVal && !normVal.isEmpty()) {
                                normalizedFieldValues.add(normVal);
                            }
                        } catch (Exception e) {
                            logger.debug(norm + " failed to normalize value: " + val);
                        }
                    }
                }
            }
        } else {
            if (logger.isTraceEnabled()) {
                logger.trace("dropping fieldname " + fieldName + " because it's not indexed.");
            }
            // drop fieldName since it isn't indexed.
            fieldNameIter.remove();
        }
    }
    this.fieldValues = normalizedFieldValues;
    if (this.fieldNames.isEmpty()) {
        throw new IllegalArgumentException("Need at least 1 indexed field to query with.");
    }
    // Generate & set the query ranges
    this.ranges = generateRanges(config);
    // Find out if we need to list unique data types.
    if (null != settings.findParameter(Constants.UNIQ_DATATYPE)) {
        this.uniqueByDataType = Boolean.parseBoolean(settings.findParameter(Constants.UNIQ_DATATYPE).getParameterValue());
        if (logger.isTraceEnabled()) {
            logger.trace("uniqueByDataType: " + uniqueByDataType);
        }
    }
    // Find out if we need to list unique visibilities.
    if (null != settings.findParameter(Constants.UNIQ_VISIBILITY)) {
        this.uniqueByVisibility = Boolean.parseBoolean(settings.findParameter(Constants.UNIQ_VISIBILITY).getParameterValue());
        if (logger.isTraceEnabled()) {
            logger.trace("uniqueByVisibility: " + uniqueByVisibility);
        }
    }
    if (logger.isTraceEnabled()) {
        logger.trace("FieldNames: ");
        for (String f : this.fieldNames) {
            logger.trace("\t" + f);
        }
        logger.trace("FieldValues: ");
        if (null == this.fieldValues) {
            logger.trace("\tnone");
        } else {
            for (String f : this.fieldValues) {
                logger.trace("\t" + f);
            }
        }
        logger.trace("uniqueByDataType: " + uniqueByDataType);
        logger.trace("uniqueByVisibility: " + uniqueByVisibility);
    }
    return config;
}

Example 23 with ScannerFactory

use of datawave.query.tables.ScannerFactory in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testGuardAgainstVeryOddlyFormedJexlQueriesLikeFoo.

@Test
public void testGuardAgainstVeryOddlyFormedJexlQueriesLikeFoo() throws Exception {
    String originalQuery = "foo";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector()), helper);
    rangeStream.streamPlans(script);
    assertEquals(IndexStream.StreamContext.UNINDEXED, rangeStream.context());
    assertEquals(Collections.emptyIterator(), rangeStream.iterator());
}

Example 24 with ScannerFactory

use of datawave.query.tables.ScannerFactory in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testNegatedNonIndexed.

@Test
public void testNegatedNonIndexed() throws Exception {
    String originalQuery = "FOO == 'bag' && TACO != 'tacocat'";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Range range1 = makeTestRange("20190314", "datatype1\u0000234");
    Range range2 = makeTestRange("20190314", "datatype1\u0000345");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector()), helper).setLimitScanners(true);
    for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}

Example 25 with ScannerFactory

use of datawave.query.tables.ScannerFactory in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testOrOneFieldIndexed.

@Test
public void testOrOneFieldIndexed() throws Exception {
    String originalQuery = "(FOO == 'bag' || TACO == 'ba')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    helper.addFields(ImmutableSet.of("TACO"));
    assertFalse(new RangeStream(config, new ScannerFactory(config.getConnector()), helper).streamPlans(script).iterator().hasNext());
}