Search in sources :

Example 1 with GeometryType

use of datawave.data.type.GeometryType in project datawave by NationalSecurityAgency.

the class ShardQueryConfigurationTest method testDeepCopyConstructor.

/**
 * Test that for a given set of collections, stored in a ShardQueryConfiguration, will in fact be deep-copied into a new ShardQueryConfiguration object.
 */
@Test
public void testDeepCopyConstructor() {
    // Instantiate a 'other' ShardQueryConfiguration
    ShardQueryConfiguration other = ShardQueryConfiguration.create();
    // Setup collections for deep copy
    List<String> realmSuffixExclusionPatterns = Lists.newArrayList("somePattern");
    SimpleDateFormat shardDateFormatter = new SimpleDateFormat("yyyyMMdd");
    List<String> enricherClassNames = Lists.newArrayList("enricherClassNameA");
    List<String> filterClassNames = Lists.newArrayList("filterClassNameA");
    List<String> indexFilteringClassNames = Lists.newArrayList("indexFilteringClassNameA");
    Set<String> nonEventKeyPrefixes = Sets.newHashSet("nonEventKeyPrefixA");
    Set<String> unevaluatedFields = Sets.newHashSet("unevaluatedFieldA");
    Set<String> dataTypeFilter = Sets.newHashSet("dataTypeFilterA");
    IndexHole indexHole = new IndexHole(new String[] { "0", "1" }, new String[] { "2", "3" });
    List<IndexHole> indexHoles = Lists.newArrayList(indexHole);
    Set<String> projectFields = Sets.newHashSet("projectFieldA");
    Set<String> blacklistedFields = Sets.newHashSet("blacklistedFieldA");
    Set<String> indexedFields = Sets.newHashSet("indexedFieldA");
    Set<String> normalizedFields = Sets.newHashSet("normalizedFieldA");
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.put("K001", new NoOpType("V"));
    Multimap<String, Type<?>> queryFieldsDatatypes = HashMultimap.create();
    queryFieldsDatatypes.put("K002", new NoOpType("V"));
    Multimap<String, Type<?>> normalizedFieldsDatatypes = HashMultimap.create();
    normalizedFieldsDatatypes.put("K003", new NoOpType("V"));
    Multimap<String, String> compositeToFieldMap = HashMultimap.create();
    compositeToFieldMap.put("K004", "V");
    Map<String, DiscreteIndexType<?>> fieldToDiscreteIndexType = Maps.newHashMap();
    fieldToDiscreteIndexType.put("GEO", new GeometryType());
    Map<String, Date> compositeTransitionDates = Maps.newHashMap();
    Date transitionDate = new Date();
    compositeTransitionDates.put("K005", transitionDate);
    Map<String, String> compositeFieldSeparators = Maps.newHashMap();
    compositeFieldSeparators.put("GEO", " ");
    Set<String> queryTermFrequencyFields = Sets.newHashSet("fieldA");
    Set<String> limitFields = Sets.newHashSet("limitFieldA");
    Map<String, String> hierarchyFieldOptions = Maps.newHashMap();
    hierarchyFieldOptions.put("K006", "V");
    List<String> documentPermutations = Lists.newArrayList(DocumentPermutation.class.getName());
    QueryModel queryModel = new QueryModel();
    QueryImpl query = new QueryImpl();
    Set<String> groupFields = Sets.newHashSet("groupFieldA");
    UniqueFields uniqueFields = new UniqueFields();
    uniqueFields.put("uniqueFieldA", UniqueGranularity.ALL);
    List<String> contentFieldNames = Lists.newArrayList("fieldA");
    Set<String> noExpansionFields = Sets.newHashSet("NoExpansionFieldA");
    Set<String> disallowedRegexPatterns = Sets.newHashSet(".*", ".*?");
    // Set collections on 'other' ShardQueryConfiguration
    other.setRealmSuffixExclusionPatterns(realmSuffixExclusionPatterns);
    other.setShardDateFormatter(shardDateFormatter);
    other.setEnricherClassNames(enricherClassNames);
    other.setFilterClassNames(filterClassNames);
    other.setIndexFilteringClassNames(indexFilteringClassNames);
    other.setNonEventKeyPrefixes(nonEventKeyPrefixes);
    other.setUnevaluatedFields(unevaluatedFields);
    other.setDatatypeFilter(dataTypeFilter);
    other.setIndexHoles(indexHoles);
    other.setProjectFields(projectFields);
    other.setBlacklistedFields(blacklistedFields);
    other.setIndexedFields(indexedFields);
    other.setNormalizedFields(normalizedFields);
    other.setDataTypes(dataTypes);
    other.setQueryFieldsDatatypes(queryFieldsDatatypes);
    other.setNormalizedFieldsDatatypes(normalizedFieldsDatatypes);
    other.setCompositeToFieldMap(compositeToFieldMap);
    other.setFieldToDiscreteIndexTypes(fieldToDiscreteIndexType);
    other.setCompositeTransitionDates(compositeTransitionDates);
    other.setCompositeFieldSeparators(compositeFieldSeparators);
    other.setQueryTermFrequencyFields(queryTermFrequencyFields);
    other.setLimitFields(limitFields);
    other.setHierarchyFieldOptions(hierarchyFieldOptions);
    other.setDocumentPermutations(documentPermutations);
    other.setQueryModel(queryModel);
    other.setQuery(query);
    other.setGroupFields(groupFields);
    other.setUniqueFields(uniqueFields);
    other.setContentFieldNames(contentFieldNames);
    other.setNoExpansionFields(noExpansionFields);
    other.setDisallowedRegexPatterns(disallowedRegexPatterns);
    // Copy 'other' ShardQueryConfiguration into a new config
    ShardQueryConfiguration config = ShardQueryConfiguration.create(other);
    // Modify original collections
    realmSuffixExclusionPatterns.add("anotherPattern");
    shardDateFormatter = new SimpleDateFormat("yyyyMMdd-mm:SS");
    enricherClassNames.add("enricherClassNameB");
    filterClassNames.add("filterClassNameB");
    indexFilteringClassNames.add("indexFilteringClassNameB");
    nonEventKeyPrefixes.add("nonEventKeyPrefixB");
    unevaluatedFields.add("unevaluatedFieldB");
    dataTypeFilter.add("dataTypeFilterB");
    IndexHole otherIndexHole = new IndexHole(new String[] { "4", "5" }, new String[] { "6", "7" });
    indexHoles.add(otherIndexHole);
    projectFields.add("projectFieldB");
    blacklistedFields.add("blacklistedFieldB");
    indexedFields.add("indexedFieldB");
    normalizedFields.add("normalizedFieldB");
    dataTypes.put("K2", new NoOpType("V2"));
    queryFieldsDatatypes.put("K", new NoOpType("V2"));
    normalizedFieldsDatatypes.put("K2", new NoOpType("V2"));
    compositeToFieldMap.put("K2", "V2");
    queryTermFrequencyFields.add("fieldB");
    limitFields.add("limitFieldB");
    hierarchyFieldOptions.put("K2", "V2");
    documentPermutations.add(DocumentProjection.class.getName());
    queryModel.addTermToModel("aliasA", "diskNameA");
    query.setId(UUID.randomUUID());
    groupFields.add("groupFieldB");
    uniqueFields.put("uniqueFieldB", UniqueGranularity.ALL);
    contentFieldNames.add("fieldB");
    disallowedRegexPatterns.add("blah");
    // Assert that copied collections were deep copied and remain unchanged
    Assert.assertEquals(Lists.newArrayList("somePattern"), config.getRealmSuffixExclusionPatterns());
    Assert.assertEquals(new SimpleDateFormat("yyyyMMdd"), config.getShardDateFormatter());
    Assert.assertEquals(Lists.newArrayList("enricherClassNameA"), config.getEnricherClassNames());
    Assert.assertEquals(Lists.newArrayList("filterClassNameA"), config.getFilterClassNames());
    Assert.assertEquals(Lists.newArrayList("indexFilteringClassNameA"), config.getIndexFilteringClassNames());
    Assert.assertEquals(Sets.newHashSet("nonEventKeyPrefixA"), config.getNonEventKeyPrefixes());
    Assert.assertEquals(Sets.newHashSet("unevaluatedFieldA"), config.getUnevaluatedFields());
    Assert.assertEquals(Sets.newHashSet("dataTypeFilterA"), config.getDatatypeFilter());
    IndexHole expectedIndexHole = new IndexHole(new String[] { "0", "1" }, new String[] { "2", "3" });
    Assert.assertEquals(Lists.newArrayList(expectedIndexHole), config.getIndexHoles());
    Assert.assertEquals(Sets.newHashSet("projectFieldA"), config.getProjectFields());
    Assert.assertEquals(Sets.newHashSet("blacklistedFieldA"), config.getBlacklistedFields());
    Assert.assertEquals(Sets.newHashSet("indexedFieldA"), config.getIndexedFields());
    // This assert is different from the setter as setNormalizedFieldsAsDatatypes will overwrite the normalizedFields with
    // a new keyset.
    Assert.assertEquals(Sets.newHashSet("K003"), config.getNormalizedFields());
    Multimap<String, Type<?>> expectedDataTypes = HashMultimap.create();
    expectedDataTypes.put("K001", new NoOpType("V"));
    Assert.assertEquals(expectedDataTypes, config.getDataTypes());
    Multimap<String, Type<?>> expectedQueryFieldsDatatypes = HashMultimap.create();
    expectedQueryFieldsDatatypes.put("K002", new NoOpType("V"));
    Assert.assertEquals(expectedQueryFieldsDatatypes, config.getQueryFieldsDatatypes());
    Multimap<String, Type<?>> expectedNormalizedFieldsDatatypes = HashMultimap.create();
    expectedNormalizedFieldsDatatypes.put("K003", new NoOpType("V"));
    Assert.assertEquals(expectedNormalizedFieldsDatatypes, config.getNormalizedFieldsDatatypes());
    Multimap<String, String> expectedCompositeToFieldMap = ArrayListMultimap.create();
    expectedCompositeToFieldMap.put("K004", "V");
    Assert.assertEquals(expectedCompositeToFieldMap, config.getCompositeToFieldMap());
    Map<String, DiscreteIndexType<?>> expectedFieldToDiscreteIndexType = Maps.newHashMap();
    expectedFieldToDiscreteIndexType.put("GEO", new GeometryType());
    Assert.assertEquals(expectedFieldToDiscreteIndexType, config.getFieldToDiscreteIndexTypes());
    Map<String, Date> expectedCompositeTransitionDates = Maps.newHashMap();
    expectedCompositeTransitionDates.put("K005", transitionDate);
    Assert.assertEquals(expectedCompositeTransitionDates, config.getCompositeTransitionDates());
    Map<String, String> expectedCompositeFieldSeparators = Maps.newHashMap();
    expectedCompositeFieldSeparators.put("GEO", " ");
    Assert.assertEquals(expectedCompositeFieldSeparators, config.getCompositeFieldSeparators());
    Assert.assertEquals(Sets.newHashSet("fieldA"), config.getQueryTermFrequencyFields());
    Assert.assertEquals(Sets.newHashSet("limitFieldA"), config.getLimitFields());
    Map<String, String> expectedHierarchyFieldOptions = Maps.newHashMap();
    expectedHierarchyFieldOptions.put("K006", "V");
    Assert.assertEquals(expectedHierarchyFieldOptions, config.getHierarchyFieldOptions());
    Assert.assertEquals(Lists.newArrayList(DocumentPermutation.class.getName()), config.getDocumentPermutations());
    QueryModel expectedQueryModel = new QueryModel();
    Assert.assertEquals(expectedQueryModel.getForwardQueryMapping(), config.getQueryModel().getForwardQueryMapping());
    Assert.assertEquals(expectedQueryModel.getReverseQueryMapping(), config.getQueryModel().getReverseQueryMapping());
    Assert.assertEquals(expectedQueryModel.getUnevaluatedFields(), config.getQueryModel().getUnevaluatedFields());
    Assert.assertEquals(Sets.newHashSet(".*", ".*?"), config.getDisallowedRegexPatterns());
    // Account for QueryImpl.duplicate() generating a random UUID on the duplicate
    QueryImpl expectedQuery = new QueryImpl();
    expectedQuery.setId(config.getQuery().getId());
    Assert.assertEquals(expectedQuery, config.getQuery());
    Assert.assertEquals(Sets.newHashSet("groupFieldA"), config.getGroupFields());
    UniqueFields expectedUniqueFields = new UniqueFields();
    expectedUniqueFields.put("uniqueFieldA", UniqueGranularity.ALL);
    Assert.assertEquals(expectedUniqueFields, config.getUniqueFields());
    Assert.assertEquals(Lists.newArrayList("fieldA"), config.getContentFieldNames());
    Assert.assertEquals(Sets.newHashSet("NoExpansionFieldA"), config.getNoExpansionFields());
}
Also used : NoOpType(datawave.data.type.NoOpType) QueryModel(datawave.query.model.QueryModel) Date(java.util.Date) DiscreteIndexType(datawave.data.type.DiscreteIndexType) GeometryType(datawave.data.type.GeometryType) DocumentPermutation(datawave.query.function.DocumentPermutation) NoOpType(datawave.data.type.NoOpType) StringType(datawave.data.type.StringType) GeometryType(datawave.data.type.GeometryType) DiscreteIndexType(datawave.data.type.DiscreteIndexType) Type(datawave.data.type.Type) DateType(datawave.data.type.DateType) QueryImpl(datawave.webservice.query.QueryImpl) UniqueFields(datawave.query.attributes.UniqueFields) SimpleDateFormat(java.text.SimpleDateFormat) DocumentProjection(datawave.query.function.DocumentProjection) Test(org.junit.Test)

Example 2 with GeometryType

use of datawave.data.type.GeometryType in project datawave by NationalSecurityAgency.

the class ExpandCompositeTermsTest method test27.

@Test
public void test27() throws Exception {
    ShardQueryConfiguration conf = new ShardQueryConfiguration();
    Multimap<String, String> compositeToFieldMap = LinkedListMultimap.create();
    compositeToFieldMap.put("GEO", "GEO");
    compositeToFieldMap.put("GEO", "WKT");
    conf.setCompositeToFieldMap(compositeToFieldMap);
    Map<String, String> compositeToSeparatorMap = new HashMap<>();
    compositeToSeparatorMap.put("GEO", ",");
    conf.setCompositeFieldSeparators(compositeToSeparatorMap);
    Set<String> indexedFields = new HashSet<>();
    indexedFields.add("GEO");
    conf.getFieldToDiscreteIndexTypes().put("GEO", new GeometryType());
    String normNum = Normalizer.NUMBER_NORMALIZER.normalize("55");
    String query = "(GEO == '0202' || (((_Bounded_ = true) && (GEO >= '030a' && GEO <= '0335')))) && WKT == '" + normNum + "'";
    String expected = "(GEO == '0202,+bE5.5' || ((((_Bounded_ = true) && (GEO >= '030a,+bE5.5' && GEO <= '0335,+bE5.5')) && ((_Eval_ = true) && (((_Bounded_ = true) && (GEO >= '030a' && GEO <= '0335')) && WKT == '+bE5.5')))))";
    runTestQuery(query, expected, indexedFields, conf);
}
Also used : GeometryType(datawave.data.type.GeometryType) HashMap(java.util.HashMap) ShardQueryConfiguration(datawave.query.config.ShardQueryConfiguration) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 3 with GeometryType

use of datawave.data.type.GeometryType in project datawave by NationalSecurityAgency.

the class ExpandCompositeTermsTest method test14.

@Test
public void test14() throws Exception {
    ShardQueryConfiguration conf = new ShardQueryConfiguration();
    Multimap<String, String> compositeToFieldMap = LinkedListMultimap.create();
    compositeToFieldMap.put("GEO", "GEO");
    compositeToFieldMap.put("GEO", "WKT_BYTE_LENGTH");
    conf.setCompositeToFieldMap(compositeToFieldMap);
    Map<String, String> compositeToSeparatorMap = new HashMap<>();
    compositeToSeparatorMap.put("GEO", ",");
    conf.setCompositeFieldSeparators(compositeToSeparatorMap);
    Set<String> indexedFields = new HashSet<>();
    indexedFields.add("GEO");
    conf.getFieldToDiscreteIndexTypes().put("GEO", new GeometryType());
    String query = "((_Bounded_ = true) && (GEO >= '0100' && GEO <= '0103')) && WKT_BYTE_LENGTH <= '" + Normalizer.NUMBER_NORMALIZER.normalize("12345") + "'";
    String expected = "((_Bounded_ = true) && (GEO >= '0100' && GEO <= '0103,+eE1.2345')) && ((_Eval_ = true) && (((_Bounded_ = true) && (GEO >= '0100' && GEO <= '0103')) && WKT_BYTE_LENGTH <= '+eE1.2345'))";
    runTestQuery(query, expected, indexedFields, conf);
}
Also used : GeometryType(datawave.data.type.GeometryType) HashMap(java.util.HashMap) ShardQueryConfiguration(datawave.query.config.ShardQueryConfiguration) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 4 with GeometryType

use of datawave.data.type.GeometryType in project datawave by NationalSecurityAgency.

the class ExpandCompositeTermsTest method test29.

@Test
public void test29() throws Exception {
    ShardQueryConfiguration conf = new ShardQueryConfiguration();
    Multimap<String, String> compositeToFieldMap = LinkedListMultimap.create();
    compositeToFieldMap.put("GEO", "GEO");
    compositeToFieldMap.put("GEO", "WKT");
    conf.setCompositeToFieldMap(compositeToFieldMap);
    Map<String, String> compositeToSeparatorMap = new HashMap<>();
    compositeToSeparatorMap.put("GEO", ",");
    conf.setCompositeFieldSeparators(compositeToSeparatorMap);
    Set<String> indexedFields = new HashSet<>();
    indexedFields.add("GEO");
    conf.getFieldToDiscreteIndexTypes().put("GEO", new GeometryType());
    String query = "(GEO == '0202' || GEO >= '030a') && WKT == '+bE5.5'";
    String expected = "((WKT == '+bE5.5' && GEO >= '030a') || GEO == '0202,+bE5.5')";
    runTestQuery(query, expected, indexedFields, conf);
}
Also used : GeometryType(datawave.data.type.GeometryType) HashMap(java.util.HashMap) ShardQueryConfiguration(datawave.query.config.ShardQueryConfiguration) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 5 with GeometryType

use of datawave.data.type.GeometryType in project datawave by NationalSecurityAgency.

the class ExpandCompositeTermsTest method test13.

@Test
public void test13() throws Exception {
    ShardQueryConfiguration conf = new ShardQueryConfiguration();
    Multimap<String, String> compositeToFieldMap = LinkedListMultimap.create();
    compositeToFieldMap.put("GEO", "GEO");
    compositeToFieldMap.put("GEO", "WKT_BYTE_LENGTH");
    conf.setCompositeToFieldMap(compositeToFieldMap);
    Map<String, String> compositeToSeparatorMap = new HashMap<>();
    compositeToSeparatorMap.put("GEO", ",");
    conf.setCompositeFieldSeparators(compositeToSeparatorMap);
    Set<String> indexedFields = new HashSet<>();
    indexedFields.add("GEO");
    conf.getFieldToDiscreteIndexTypes().put("GEO", new GeometryType());
    String query = "((_Bounded_ = true) && (GEO >= '0100' && GEO <= '0103')) && WKT_BYTE_LENGTH >= '" + Normalizer.NUMBER_NORMALIZER.normalize("0") + "'";
    String expected = "((_Bounded_ = true) && (GEO >= '0100,+AE0' && GEO < '0104')) && ((_Eval_ = true) && (((_Bounded_ = true) && (GEO >= '0100' && GEO <= '0103')) && WKT_BYTE_LENGTH >= '+AE0'))";
    runTestQuery(query, expected, indexedFields, conf);
}
Also used : GeometryType(datawave.data.type.GeometryType) HashMap(java.util.HashMap) ShardQueryConfiguration(datawave.query.config.ShardQueryConfiguration) HashSet(java.util.HashSet) Test(org.junit.Test)

Aggregations

GeometryType (datawave.data.type.GeometryType)18 Test (org.junit.Test)18 ShardQueryConfiguration (datawave.query.config.ShardQueryConfiguration)17 HashSet (java.util.HashSet)17 HashMap (java.util.HashMap)14 Date (java.util.Date)4 DateType (datawave.data.type.DateType)1 DiscreteIndexType (datawave.data.type.DiscreteIndexType)1 NoOpType (datawave.data.type.NoOpType)1 StringType (datawave.data.type.StringType)1 Type (datawave.data.type.Type)1 UniqueFields (datawave.query.attributes.UniqueFields)1 DocumentPermutation (datawave.query.function.DocumentPermutation)1 DocumentProjection (datawave.query.function.DocumentProjection)1 QueryModel (datawave.query.model.QueryModel)1 QueryImpl (datawave.webservice.query.QueryImpl)1 SimpleDateFormat (java.text.SimpleDateFormat)1