Search in sources :

Example 6 with MetadataHelper

use of datawave.query.util.MetadataHelper in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method initializeRangeStream.

/**
 * Initializes the range stream, whether it is configured to be a different class than the Default Range stream or not.
 *
 * @param config
 * @param scannerFactory
 * @param metadataHelper
 * @return
 */
private RangeStream initializeRangeStream(ShardQueryConfiguration config, ScannerFactory scannerFactory, MetadataHelper metadataHelper) {
    Class<? extends RangeStream> rstream;
    try {
        rstream = Class.forName(rangeStreamClass).asSubclass(RangeStream.class);
        RangeStream stream = rstream.getConstructor(ShardQueryConfiguration.class, ScannerFactory.class, MetadataHelper.class).newInstance(config, scannerFactory, metadataHelper);
        return stream.setUidIntersector(uidIntersector).setLimitScanners(limitScanners).setCreateUidsIteratorClass(createUidsIteratorClass);
    } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) {
        throw new RuntimeException(e);
    }
}
Also used : AccumuloSecurityException(org.apache.accumulo.core.client.AccumuloSecurityException) ScannerFactory(datawave.query.tables.ScannerFactory) InvocationTargetException(java.lang.reflect.InvocationTargetException) MetadataHelper(datawave.query.util.MetadataHelper) RangeStream(datawave.query.index.lookup.RangeStream) ShardQueryConfiguration(datawave.query.config.ShardQueryConfiguration)

Example 7 with MetadataHelper

use of datawave.query.util.MetadataHelper in project datawave by NationalSecurityAgency.

the class ExecutableExpansionVisitorTest method testSingleOrNonExecutableCantFix.

@Test
public void testSingleOrNonExecutableCantFix() throws Exception {
    ASTJexlScript queryTree = JexlASTHelper.parseJexlQuery("BIRTH_DATE =='123' && (filter:includeRegex(QUOTE, '.*kind.*') || BIRTH_DATE == '234')");
    // grab the reference to the QUOTE=='kind' eqnode
    JexlNode quoteNode = queryTree.jjtGetChild(0).jjtGetChild(1).jjtGetChild(0).jjtGetChild(0).jjtGetChild(0);
    ASTOrNode newOr = new ASTOrNode(ParserTreeConstants.JJTORNODE);
    newOr.jjtAddChild(quoteNode, 0);
    quoteNode.jjtSetParent(newOr);
    // attach the new or node
    queryTree.jjtGetChild(0).jjtGetChild(1).jjtGetChild(0).jjtAddChild(newOr, 0);
    newOr.jjtSetParent(queryTree.jjtGetChild(0).jjtGetChild(1).jjtGetChild(0));
    ShardQueryConfiguration config = EasyMock.createMock(ShardQueryConfiguration.class);
    MetadataHelper helper = EasyMock.createMock(MetadataHelper.class);
    HashSet<String> indexedFields = new HashSet<>();
    indexedFields.add("UUID");
    indexedFields.add("QUOTE");
    EasyMock.expect(config.getIndexedFields()).andReturn(indexedFields).anyTimes();
    EasyMock.replay(config, helper);
    ASTJexlScript newTree = ExecutableExpansionVisitor.expand(queryTree, config, helper);
    EasyMock.verify(config, helper);
    // starts executable
    Assert.assertFalse(ExecutableDeterminationVisitor.isExecutable(queryTree, config, helper));
    // what came out is executable
    Assert.assertFalse(ExecutableDeterminationVisitor.isExecutable(newTree, config, helper));
    // the visitor changed nothing
    Assert.assertTrue(JexlStringBuildingVisitor.buildQuery(newTree), JexlStringBuildingVisitor.buildQuery(newTree).equals(JexlStringBuildingVisitor.buildQuery(queryTree)));
}
Also used : ASTOrNode(org.apache.commons.jexl2.parser.ASTOrNode) MetadataHelper(datawave.query.util.MetadataHelper) ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) ExceededValueThresholdMarkerJexlNode(datawave.query.jexl.nodes.ExceededValueThresholdMarkerJexlNode) ExceededOrThresholdMarkerJexlNode(datawave.query.jexl.nodes.ExceededOrThresholdMarkerJexlNode) JexlNode(org.apache.commons.jexl2.parser.JexlNode) ShardQueryConfiguration(datawave.query.config.ShardQueryConfiguration) HashSet(java.util.HashSet) CompositeFunctionsTest(datawave.query.CompositeFunctionsTest) Test(org.junit.Test)

Example 8 with MetadataHelper

use of datawave.query.util.MetadataHelper in project datawave by NationalSecurityAgency.

the class ExecutableExpansionVisitorTest method testExceededThresholdExpansionExternal.

@Test
public void testExceededThresholdExpansionExternal() throws Exception {
    ASTJexlScript queryTree = JexlASTHelper.parseJexlQuery("UUID == 'capone' && (filter:includeRegex(QUOTE,'.*kind.*') || QUOTE == 'kind' || BIRTH_DATE == '123')");
    // update the generated queryTree to have an ExceededThreshold marker
    JexlNode child = new ExceededValueThresholdMarkerJexlNode(queryTree.jjtGetChild(0).jjtGetChild(0));
    // unlink the old node
    queryTree.jjtGetChild(0).jjtGetChild(0).jjtSetParent(null);
    // overwrite the old UUID==capone with the ExceededThreshold marker
    queryTree.jjtGetChild(0).jjtAddChild(child, 0);
    ShardQueryConfiguration config = EasyMock.createMock(ShardQueryConfiguration.class);
    MetadataHelper helper = EasyMock.createMock(MetadataHelper.class);
    HashSet<String> indexedFields = new HashSet<>();
    indexedFields.add("UUID");
    indexedFields.add("QUOTE");
    EasyMock.expect(config.getIndexedFields()).andReturn(indexedFields).anyTimes();
    EasyMock.replay(config, helper);
    ASTJexlScript newTree = ExecutableExpansionVisitor.expand(queryTree, config, helper);
    EasyMock.verify(config, helper);
    // included ExceededValueThresholdMarker before
    Assert.assertTrue(JexlStringBuildingVisitor.buildQuery(queryTree), JexlStringBuildingVisitor.buildQuery(queryTree).equals("((_Value_ = true) && (UUID == 'capone')) && (filter:includeRegex(QUOTE, '.*kind.*') || QUOTE == 'kind' || BIRTH_DATE == '123')"));
    // not executable
    Assert.assertFalse(ExecutableDeterminationVisitor.isExecutable(queryTree, config, helper));
    // what came out is executable
    Assert.assertTrue(ExecutableDeterminationVisitor.isExecutable(newTree, config, helper));
    // it looks like what we'd expect
    String expected = "(QUOTE == 'kind' && ((_Value_ = true) && (UUID == 'capone'))) || " + "((filter:includeRegex(QUOTE, '.*kind.*') || BIRTH_DATE == '123') && ((_Value_ = true) && (UUID == 'capone')))";
    Assert.assertEquals(expected, JexlStringBuildingVisitor.buildQueryWithoutParse(newTree));
}
Also used : MetadataHelper(datawave.query.util.MetadataHelper) ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) ExceededValueThresholdMarkerJexlNode(datawave.query.jexl.nodes.ExceededValueThresholdMarkerJexlNode) ExceededOrThresholdMarkerJexlNode(datawave.query.jexl.nodes.ExceededOrThresholdMarkerJexlNode) JexlNode(org.apache.commons.jexl2.parser.JexlNode) ExceededValueThresholdMarkerJexlNode(datawave.query.jexl.nodes.ExceededValueThresholdMarkerJexlNode) ShardQueryConfiguration(datawave.query.config.ShardQueryConfiguration) HashSet(java.util.HashSet) CompositeFunctionsTest(datawave.query.CompositeFunctionsTest) Test(org.junit.Test)

Example 9 with MetadataHelper

use of datawave.query.util.MetadataHelper in project datawave by NationalSecurityAgency.

the class ExecutableExpansionVisitorTest method testArbitraryNodeExpansionFlatten.

@Test
public void testArbitraryNodeExpansionFlatten() throws Exception {
    ASTJexlScript queryTree = JexlASTHelper.parseJexlQuery("UUID == 'capone' && (filter:includeRegex(QUOTE,'.*kind.*') || QUOTE == 'kind' || BIRTH_DATE == '123')");
    ShardQueryConfiguration config = EasyMock.createMock(ShardQueryConfiguration.class);
    MetadataHelper helper = EasyMock.createMock(MetadataHelper.class);
    HashSet<String> indexedFields = new HashSet<>();
    indexedFields.add("UUID");
    indexedFields.add("QUOTE");
    EasyMock.expect(config.getIndexedFields()).andReturn(indexedFields).anyTimes();
    EasyMock.replay(config, helper);
    // find an orNode in the tree
    ExecutableExpansionVisitor visitor = new ExecutableExpansionVisitor(config, helper);
    ASTJexlScript rebuilt = TreeFlatteningRebuildingVisitor.flatten(queryTree);
    visitor.visit(rebuilt.jjtGetChild(0), null);
    EasyMock.verify(config, helper);
    String expected = "(QUOTE == 'kind' && UUID == 'capone') || ((filter:includeRegex(QUOTE, '.*kind.*') || BIRTH_DATE == '123') && UUID == 'capone')";
    Assert.assertEquals(expected, JexlStringBuildingVisitor.buildQuery(rebuilt));
}
Also used : MetadataHelper(datawave.query.util.MetadataHelper) ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) ShardQueryConfiguration(datawave.query.config.ShardQueryConfiguration) HashSet(java.util.HashSet) CompositeFunctionsTest(datawave.query.CompositeFunctionsTest) Test(org.junit.Test)

Example 10 with MetadataHelper

use of datawave.query.util.MetadataHelper in project datawave by NationalSecurityAgency.

the class ExecutableExpansionVisitorTest method testExceededOrThresholdCannotExpand.

@Test
public void testExceededOrThresholdCannotExpand() throws Exception {
    ASTJexlScript queryTree = JexlASTHelper.parseJexlQuery("UUID == 'capone' && (((_List_ = true) && (((id = 'some-bogus-id') && (field = 'QUOTE') && (params = '{\"values\":[\"a\",\"b\",\"c\"]}')))))");
    ShardQueryConfiguration config = EasyMock.createMock(ShardQueryConfiguration.class);
    MetadataHelper helper = EasyMock.createMock(MetadataHelper.class);
    HashSet<String> indexedFields = new HashSet<>();
    indexedFields.add("UUID");
    indexedFields.add("QUOTE");
    EasyMock.expect(config.getIndexedFields()).andReturn(indexedFields).anyTimes();
    EasyMock.replay(config, helper);
    ASTJexlScript newTree = ExecutableExpansionVisitor.expand(queryTree, config, helper);
    EasyMock.verify(config, helper);
    // included ExceededValueThresholdMarker before
    Assert.assertTrue(JexlStringBuildingVisitor.buildQuery(queryTree), JexlStringBuildingVisitor.buildQuery(queryTree).equals("UUID == 'capone' && (((_List_ = true) && (((id = 'some-bogus-id') && (field = 'QUOTE') && (params = '{\"values\":[\"a\",\"b\",\"c\"]}')))))"));
    // starts off executable
    Assert.assertTrue(ExecutableDeterminationVisitor.isExecutable(queryTree, config, helper));
    // what came out is executable
    Assert.assertTrue(ExecutableDeterminationVisitor.isExecutable(newTree, config, helper));
    // the visitor changed nothing
    Assert.assertTrue(JexlStringBuildingVisitor.buildQuery(newTree), JexlStringBuildingVisitor.buildQuery(newTree).equals(JexlStringBuildingVisitor.buildQuery(queryTree)));
}
Also used : MetadataHelper(datawave.query.util.MetadataHelper) ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) ShardQueryConfiguration(datawave.query.config.ShardQueryConfiguration) HashSet(java.util.HashSet) CompositeFunctionsTest(datawave.query.CompositeFunctionsTest) Test(org.junit.Test)

Aggregations

MetadataHelper (datawave.query.util.MetadataHelper)21 ShardQueryConfiguration (datawave.query.config.ShardQueryConfiguration)16 HashSet (java.util.HashSet)14 ASTJexlScript (org.apache.commons.jexl2.parser.ASTJexlScript)14 CompositeFunctionsTest (datawave.query.CompositeFunctionsTest)13 Test (org.junit.Test)13 ExceededOrThresholdMarkerJexlNode (datawave.query.jexl.nodes.ExceededOrThresholdMarkerJexlNode)5 ExceededValueThresholdMarkerJexlNode (datawave.query.jexl.nodes.ExceededValueThresholdMarkerJexlNode)5 JexlNode (org.apache.commons.jexl2.parser.JexlNode)5 Type (datawave.data.type.Type)3 ScannerFactory (datawave.query.tables.ScannerFactory)2 QueryException (datawave.webservice.query.exception.QueryException)2 Date (java.util.Date)2 AccumuloSecurityException (org.apache.accumulo.core.client.AccumuloSecurityException)2 ASTOrNode (org.apache.commons.jexl2.parser.ASTOrNode)2 HashMultimap (com.google.common.collect.HashMultimap)1 Multimap (com.google.common.collect.Multimap)1 EdgeExtendedSummaryConfiguration (datawave.query.config.EdgeExtendedSummaryConfiguration)1 ShardIndexQueryConfiguration (datawave.query.config.ShardIndexQueryConfiguration)1 DatawaveKey (datawave.query.data.parsers.DatawaveKey)1