Search in sources :

Example 1 with NumberType

use of datawave.data.type.NumberType in project datawave by NationalSecurityAgency.

the class FunctionalSet method getGroupsForValue.

public Object getGroupsForValue(String value) {
    Set<String> groups = Sets.newHashSet();
    for (Iterator<T> iterator = iterator(); iterator.hasNext(); ) {
        ValueTuple next = iterator.next();
        String field = next.first();
        Object val = next.getValue();
        if (val instanceof String) {
            if (((String) val).equals(value)) {
                groups.add(field.substring(field.indexOf('.') + 1));
            }
        } else if (val instanceof Number) {
            String longValue = ((Number) val).toString();
            if (longValue.equals(value)) {
                groups.add(field.substring(field.indexOf('.') + 1));
            }
        } else if (val instanceof NumberType) {
            String longValue = "" + ((NumberType) val).getDelegate().longValue();
            if (longValue.equals(value)) {
                groups.add(field.substring(field.indexOf('.') + 1));
            }
        }
    }
    return !groups.isEmpty() ? groups.iterator().next() : null;
}
Also used : NumberType(datawave.data.type.NumberType) ValueTuple(datawave.query.attributes.ValueTuple)

Example 2 with NumberType

use of datawave.data.type.NumberType in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testBothIndexedPrune.

@Test
public void testBothIndexedPrune() throws Exception {
    String originalQuery = "(FOO == 'barter' || FOO == 'baggy')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Range range1 = makeTestRange("20190314_1", "datatype1\u0000123");
    Range range2 = makeTestRange("20190314_1", "datatype1\u0000345");
    Range range3 = makeTestRange("20190414_1", "datatype1\u0000123");
    Range range4 = makeTestRange("20190414_1", "datatype1\u0000345");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3, range4);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper).setLimitScanners(true);
    for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) ScannerFactory(datawave.query.tables.ScannerFactory) QueryPlan(datawave.query.planner.QueryPlan) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) NumberType(datawave.data.type.NumberType) Test(org.junit.Test)

Example 3 with NumberType

use of datawave.data.type.NumberType in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testAndOneFieldIndexed.

@Test
public void testAndOneFieldIndexed() throws Exception {
    String originalQuery = "(FOO == 'bag' && TACO == 'ba')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    helper.addFields(ImmutableSet.of("TACO"));
    Range range1 = makeTestRange("20190314", "datatype1\u0000234");
    Range range2 = makeTestRange("20190314", "datatype1\u0000345");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector()), helper);
    for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) ScannerFactory(datawave.query.tables.ScannerFactory) QueryPlan(datawave.query.planner.QueryPlan) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) NumberType(datawave.data.type.NumberType) Test(org.junit.Test)

Example 4 with NumberType

use of datawave.data.type.NumberType in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testShardAndDaysHint.

@Test
public void testShardAndDaysHint() throws Exception {
    String originalQuery = "(FOO == 'bardy') && (SHARDS_AND_DAYS = '20190314_2,20190314_1')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Range range1 = makeTestRange("20190314_1", "datatype1\u0000345");
    Range range2 = makeTestRange("20190314_1", "datatype1\u0000456");
    Range range3 = makeTestRange("20190314_1", "datatype1\u0000567");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3);
    for (QueryPlan queryPlan : new RangeStream(config, new ScannerFactory(config.getConnector()), helper).streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) QueryPlan(datawave.query.planner.QueryPlan) ScannerFactory(datawave.query.tables.ScannerFactory) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) NumberType(datawave.data.type.NumberType) Test(org.junit.Test)

Example 5 with NumberType

use of datawave.data.type.NumberType in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testOrBothIndexed.

@Test
public void testOrBothIndexed() throws Exception {
    String originalQuery = "(FOO == 'bag' || FOO == 'ba')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Range range1 = makeTestRange("20190314", "datatype1\u0000234");
    Range range2 = makeTestRange("20190314", "datatype1\u0000345");
    Range range3 = makeTestRange("20190314", "datatype1\u0000123");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3);
    for (QueryPlan queryPlan : new RangeStream(config, new ScannerFactory(config.getConnector()), helper).streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) QueryPlan(datawave.query.planner.QueryPlan) ScannerFactory(datawave.query.tables.ScannerFactory) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) NumberType(datawave.data.type.NumberType) Test(org.junit.Test)

Aggregations

NumberType (datawave.data.type.NumberType)28 LcNoDiacriticsType (datawave.data.type.LcNoDiacriticsType)21 Test (org.junit.Test)21 Type (datawave.data.type.Type)20 NoOpType (datawave.data.type.NoOpType)18 ScannerFactory (datawave.query.tables.ScannerFactory)13 MockMetadataHelper (datawave.query.util.MockMetadataHelper)13 ASTJexlScript (org.apache.commons.jexl2.parser.ASTJexlScript)13 RangeFactoryForTests.makeTestRange (datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange)9 QueryPlan (datawave.query.planner.QueryPlan)9 Range (org.apache.accumulo.core.data.Range)9 LcType (datawave.data.type.LcType)8 IpAddressType (datawave.data.type.IpAddressType)4 ShardQueryConfiguration (datawave.query.config.ShardQueryConfiguration)3 Key (org.apache.accumulo.core.data.Key)3 ColumnVisibility (org.apache.accumulo.core.security.ColumnVisibility)3 ImmutableMap (com.google.common.collect.ImmutableMap)2 MarkingFunctions (datawave.marking.MarkingFunctions)2 ValueTuple (datawave.query.attributes.ValueTuple)2 QueryIterator (datawave.query.iterator.QueryIterator)2