Examples with MetadataHelper datawave.query.util.MetadataHelper used on opensource projects

Example 16 with MetadataHelper

use of datawave.query.util.MetadataHelper in project datawave by NationalSecurityAgency.

the class ExecutableExpansionVisitorTest method testMinimumExpansionParse.

@Test
public void testMinimumExpansionParse() throws Exception {
    ASTJexlScript queryTree = JexlASTHelper.parseJexlQuery("UUID == 'capone' && (filter:includeRegex(QUOTE,'.*kind.*') || QUOTE == 'kind' || BIRTH_DATE == '123')");
    ShardQueryConfiguration config = EasyMock.createMock(ShardQueryConfiguration.class);
    MetadataHelper helper = EasyMock.createMock(MetadataHelper.class);
    HashSet<String> indexedFields = new HashSet<>();
    indexedFields.add("UUID");
    indexedFields.add("QUOTE");
    EasyMock.expect(config.getIndexedFields()).andReturn(indexedFields).anyTimes();
    EasyMock.replay(config, helper);
    ASTJexlScript newTree = ExecutableExpansionVisitor.expand(queryTree, config, helper);
    EasyMock.verify(config, helper);
    Assert.assertFalse(JexlStringBuildingVisitor.buildQuery(queryTree).equals(JexlStringBuildingVisitor.buildQuery(newTree)));
    String expected = "(QUOTE == 'kind' && UUID == 'capone') || ((filter:includeRegex(QUOTE, '.*kind.*') || BIRTH_DATE == '123') && UUID == 'capone')";
    Assert.assertEquals(expected, JexlStringBuildingVisitor.buildQuery(newTree));
}

Example 17 with MetadataHelper

use of datawave.query.util.MetadataHelper in project datawave by NationalSecurityAgency.

the class ExecutableExpansionVisitorTest method testDelayedDoubleExpansion.

@Test
public void testDelayedDoubleExpansion() throws Exception {
    ASTJexlScript queryTree = JexlASTHelper.parseJexlQuery("UUID == 'capone' && (((_Delayed_ = true) && QUOTE == 'kind') || " + "((_Delayed_ = true) && BIRTH_DATE == '123'))");
    ShardQueryConfiguration config = EasyMock.createMock(ShardQueryConfiguration.class);
    MetadataHelper helper = EasyMock.createMock(MetadataHelper.class);
    HashSet<String> indexedFields = new HashSet<>();
    indexedFields.add("UUID");
    indexedFields.add("QUOTE");
    EasyMock.expect(config.getIndexedFields()).andReturn(indexedFields).anyTimes();
    Set<String> dataTypes = new HashSet<>();
    dataTypes.add("test");
    // QUOTE being delayed creates a query that is non-executable we cannot delay a field which is nonEvent
    Set<String> nonEventFields = new HashSet<>();
    nonEventFields.add("QUOTE");
    EasyMock.expect(config.getDatatypeFilter()).andReturn(dataTypes).anyTimes();
    EasyMock.expect(helper.getNonEventFields(dataTypes)).andReturn(nonEventFields).anyTimes();
    EasyMock.replay(config, helper);
    ASTJexlScript newTree = ExecutableExpansionVisitor.expand(queryTree, config, helper);
    EasyMock.verify(config, helper);
    // included ExceededValueThresholdMarker before
    Assert.assertTrue(JexlStringBuildingVisitor.buildQuery(queryTree), JexlStringBuildingVisitor.buildQuery(queryTree).equals("UUID == 'capone' && (((_Delayed_ = true) && QUOTE == 'kind') || " + "((_Delayed_ = true) && BIRTH_DATE == '123'))"));
    // starts off executable
    Assert.assertTrue(ExecutableDeterminationVisitor.isExecutable(queryTree, config, helper));
    // what came out is executable
    Assert.assertTrue(ExecutableDeterminationVisitor.isExecutable(newTree, config, helper));
    // the visitor changed nothing
    Assert.assertTrue(JexlStringBuildingVisitor.buildQuery(newTree), JexlStringBuildingVisitor.buildQuery(newTree).equals(JexlStringBuildingVisitor.buildQuery(queryTree)));
}

Example 18 with MetadataHelper

use of datawave.query.util.MetadataHelper in project datawave by NationalSecurityAgency.

the class ExecutableExpansionVisitorTest method testArbitraryNodeExpansionFailNoFlatten.

@Test
public void testArbitraryNodeExpansionFailNoFlatten() throws Exception {
    ASTJexlScript queryTree = JexlASTHelper.parseJexlQuery("UUID == 'capone' && (filter:includeRegex(QUOTE,'.*kind.*') || QUOTE == 'kind' || BIRTH_DATE == '123')");
    ShardQueryConfiguration config = EasyMock.createMock(ShardQueryConfiguration.class);
    MetadataHelper helper = EasyMock.createMock(MetadataHelper.class);
    HashSet<String> indexedFields = new HashSet<>();
    indexedFields.add("UUID");
    indexedFields.add("QUOTE");
    EasyMock.expect(config.getIndexedFields()).andReturn(indexedFields).anyTimes();
    EasyMock.replay(config, helper);
    // find an orNode in the tree
    ExecutableExpansionVisitor visitor = new ExecutableExpansionVisitor(config, helper);
    Object data = visitor.visit(queryTree.jjtGetChild(0), null);
    EasyMock.verify(config, helper);
    Assert.assertFalse(data instanceof ExecutableExpansionVisitor.ExpansionTracker);
}

Example 19 with MetadataHelper

use of datawave.query.util.MetadataHelper in project datawave by NationalSecurityAgency.

the class ExecutableExpansionVisitorTest method testNoReferenceOrReferenceExpressions.

@Test
public void testNoReferenceOrReferenceExpressions() throws Exception {
    ASTJexlScript queryTree = JexlASTHelper.parseJexlQuery("UUID == 'capone' && (filter:includeRegex(QUOTE,'.*kind.*') || QUOTE == 'kind' || BIRTH_DATE == '123')");
    // strip reference/referenceExpressions
    queryTree = TreeFlatteningRebuildingVisitor.flattenAll(queryTree);
    ShardQueryConfiguration config = EasyMock.createMock(ShardQueryConfiguration.class);
    MetadataHelper helper = EasyMock.createMock(MetadataHelper.class);
    HashSet<String> indexedFields = new HashSet<>();
    indexedFields.add("UUID");
    indexedFields.add("QUOTE");
    EasyMock.expect(config.getIndexedFields()).andReturn(indexedFields).anyTimes();
    EasyMock.replay(config, helper);
    ASTJexlScript newTree = ExecutableExpansionVisitor.expand(queryTree, config, helper);
    EasyMock.verify(config, helper);
    // starts executable
    Assert.assertFalse(ExecutableDeterminationVisitor.isExecutable(queryTree, config, helper));
    // what came out is executable
    Assert.assertTrue(ExecutableDeterminationVisitor.isExecutable(newTree, config, helper));
    // the visitor changed nothing
    Assert.assertTrue(JexlStringBuildingVisitor.buildQuery(newTree), JexlStringBuildingVisitor.buildQuery(newTree).equals("(QUOTE == 'kind' && UUID == 'capone') || " + "((filter:includeRegex(QUOTE, '.*kind.*') || BIRTH_DATE == '123') && UUID == 'capone')"));
}

Example 20 with MetadataHelper

use of datawave.query.util.MetadataHelper in project datawave by NationalSecurityAgency.

the class FieldIndexCountQueryLogic method initialize.

@Override
public GenericQueryConfiguration initialize(Connector connection, Query settings, Set<Authorizations> auths) throws Exception {
    if (logger.isTraceEnabled()) {
        logger.trace("initialize");
    }
    this.scannerFactory = new ScannerFactory(connection);
    MetadataHelper metadataHelper = prepareMetadataHelper(connection, this.getMetadataTableName(), auths);
    String modelName = this.getModelName();
    String modelTableName = this.getModelTableName();
    // Check if the default modelName and modelTableNames have been overriden by custom parameters.
    if (null != settings.findParameter(QueryParameters.PARAMETER_MODEL_NAME) && !settings.findParameter(QueryParameters.PARAMETER_MODEL_NAME).getParameterValue().trim().isEmpty()) {
        modelName = settings.findParameter(QueryParameters.PARAMETER_MODEL_NAME).getParameterValue().trim();
    }
    if (null != settings.findParameter(QueryParameters.PARAMETER_MODEL_TABLE_NAME) && !settings.findParameter(QueryParameters.PARAMETER_MODEL_TABLE_NAME).getParameterValue().trim().isEmpty()) {
        modelTableName = settings.findParameter(QueryParameters.PARAMETER_MODEL_TABLE_NAME).getParameterValue().trim();
    }
    if (null != modelName && null == modelTableName) {
        throw new IllegalArgumentException(QueryParameters.PARAMETER_MODEL_NAME + " has been specified but " + QueryParameters.PARAMETER_MODEL_TABLE_NAME + " is missing. Both are required to use a model");
    }
    if (null != modelName && null != modelTableName) {
        this.queryModel = metadataHelper.getQueryModel(modelTableName, modelName, this.getUnevaluatedFields());
    }
    // I'm using this config object in a pinch, we should probably create a custom one.
    ShardQueryConfiguration config = ShardQueryConfiguration.create(this, settings);
    config.setConnector(connection);
    config.setAuthorizations(auths);
    // the following throw IllegalArgumentExceptions if validation fails.
    parseQuery(config, settings);
    configDate(config, settings);
    configTypeFilter(config, settings);
    Set<String> normalizedFieldValues = null;
    Iterator<String> fieldNameIter = fieldNames.iterator();
    while (fieldNameIter.hasNext()) {
        String fieldName = fieldNameIter.next();
        // check that the field name is actually an indexed field
        Set<Type<?>> normalizerSet = metadataHelper.getDatatypesForField(fieldName, config.getDatatypeFilter());
        if (null != normalizerSet && !normalizerSet.isEmpty()) {
            if (null != this.fieldValues && !this.fieldValues.isEmpty()) {
                for (Type<?> norm : normalizerSet) {
                    if (null == normalizedFieldValues) {
                        normalizedFieldValues = new HashSet<>();
                    }
                    for (String val : this.fieldValues) {
                        try {
                            String normVal = norm.normalize(val);
                            if (null != normVal && !normVal.isEmpty()) {
                                normalizedFieldValues.add(normVal);
                            }
                        } catch (Exception e) {
                            logger.debug(norm + " failed to normalize value: " + val);
                        }
                    }
                }
            }
        } else {
            if (logger.isTraceEnabled()) {
                logger.trace("dropping fieldname " + fieldName + " because it's not indexed.");
            }
            // drop fieldName since it isn't indexed.
            fieldNameIter.remove();
        }
    }
    this.fieldValues = normalizedFieldValues;
    if (this.fieldNames.isEmpty()) {
        throw new IllegalArgumentException("Need at least 1 indexed field to query with.");
    }
    // Generate & set the query ranges
    this.ranges = generateRanges(config);
    // Find out if we need to list unique data types.
    if (null != settings.findParameter(Constants.UNIQ_DATATYPE)) {
        this.uniqueByDataType = Boolean.parseBoolean(settings.findParameter(Constants.UNIQ_DATATYPE).getParameterValue());
        if (logger.isTraceEnabled()) {
            logger.trace("uniqueByDataType: " + uniqueByDataType);
        }
    }
    // Find out if we need to list unique visibilities.
    if (null != settings.findParameter(Constants.UNIQ_VISIBILITY)) {
        this.uniqueByVisibility = Boolean.parseBoolean(settings.findParameter(Constants.UNIQ_VISIBILITY).getParameterValue());
        if (logger.isTraceEnabled()) {
            logger.trace("uniqueByVisibility: " + uniqueByVisibility);
        }
    }
    if (logger.isTraceEnabled()) {
        logger.trace("FieldNames: ");
        for (String f : this.fieldNames) {
            logger.trace("\t" + f);
        }
        logger.trace("FieldValues: ");
        if (null == this.fieldValues) {
            logger.trace("\tnone");
        } else {
            for (String f : this.fieldValues) {
                logger.trace("\t" + f);
            }
        }
        logger.trace("uniqueByDataType: " + uniqueByDataType);
        logger.trace("uniqueByVisibility: " + uniqueByVisibility);
    }
    return config;
}