use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.
the class QueryModelVisitorTest method testStillExpandIntoExcludedField.
// NoExpansion only excludes fields in the original query. A query can still expand into an excluded field
@Test
public void testStillExpandIntoExcludedField() {
QueryModel model = new QueryModel();
model.addTermToModel("FIELD_A", "FIELD_B");
model.addTermToModel("FIELD_A", "FIELD_C");
String query = "FIELD_A == 'bar' && filter:noExpansion(FIELD_C)";
String expected = "(FIELD_B == 'bar' || FIELD_C == 'bar')";
testNoExpansion(query, expected, model, Sets.newHashSet("FIELD_C"));
}
use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.
the class QueryModelVisitorTest method testModelExpansionWithNoExpansionFunction.
@Test
public void testModelExpansionWithNoExpansionFunction() {
// model contains expansions for both FIELD_A and FIELD_B, presence of filter:noExpansion(FIELD_B) prevents
// that portion of the model expansion.
QueryModel model = new QueryModel();
model.addTermToModel("FIELD_A", "FIELD_AA");
model.addTermToModel("FIELD_A", "FIELD_AB");
model.addTermToModel("FIELD_B", "FIELD_BB");
model.addTermToModel("FIELD_B", "FIELD_BC");
// base case, both original fields are expanded
String query = "FIELD_A == 'bar' && FIELD_B == 'baz'";
String expected = "(FIELD_AA == 'bar' || FIELD_AB == 'bar') && (FIELD_BB == 'baz' || FIELD_BC == 'baz')";
testNoExpansion(query, expected, model, Collections.emptySet());
// only FIELD_B is expanded
query = "FIELD_A == 'bar' && FIELD_B == 'baz' && filter:noExpansion(FIELD_A)";
expected = "FIELD_A == 'bar' && (FIELD_BB == 'baz' || FIELD_BC == 'baz')";
testNoExpansion(query, expected, model, Sets.newHashSet("FIELD_A"));
// only FIELD_A is expanded
query = "FIELD_A == 'bar' && FIELD_B == 'baz' && filter:noExpansion(FIELD_B)";
expected = "(FIELD_AB == 'bar' || FIELD_AA == 'bar') && FIELD_B == 'baz'";
testNoExpansion(query, expected, model, Sets.newHashSet("FIELD_B"));
// neither field is expanded
query = "FIELD_A == 'bar' && FIELD_B == 'baz' && filter:noExpansion(FIELD_A,FIELD_B)";
expected = "FIELD_A == 'bar' && FIELD_B == 'baz'";
testNoExpansion(query, expected, model, Sets.newHashSet("FIELD_A", "FIELD_B"));
// both fields are expanded, NoExpansion function specified a field that does not exist in the query
query = "FIELD_A == 'bar' && FIELD_B == 'baz' && filter:noExpansion(FIELD_X,FIELD_Y)";
expected = "(FIELD_AA == 'bar' || FIELD_AB == 'bar') && (FIELD_BB == 'baz' || FIELD_BC == 'baz')";
testNoExpansion(query, expected, model, Sets.newHashSet("FIELD_X", "FIELD_Y"));
query = "FIELD_A == 'bar' && FIELD_B == 'baz' && filter:noExpansion(FIELD_X,FIELD_Y,FIELD_A)";
expected = "FIELD_A == 'bar' && (FIELD_BB == 'baz' || FIELD_BC == 'baz')";
testNoExpansion(query, expected, model, Sets.newHashSet("FIELD_A", "FIELD_X", "FIELD_Y"));
query = "FIELD_A == 'bar' && FIELD_B == 'baz' && filter:noExpansion(FIELD_A,FIELD_X,FIELD_Y)";
expected = "FIELD_A == 'bar' && (FIELD_BB == 'baz' || FIELD_BC == 'baz')";
testNoExpansion(query, expected, model, Sets.newHashSet("FIELD_A", "FIELD_X", "FIELD_Y"));
}
use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.
the class TestCardinalityConfiguration method init.
@Before
public void init() throws Exception {
URL mUrl = TestCardinalityConfiguration.class.getResource("/models/CardinalityModel.xml");
JAXBContext ctx = JAXBContext.newInstance(datawave.webservice.model.Model.class);
Unmarshaller u = ctx.createUnmarshaller();
Model MODEL = (datawave.webservice.model.Model) u.unmarshal(mUrl);
QUERY_MODEL = new QueryModel();
for (FieldMapping mapping : MODEL.getFields()) {
if (mapping.getDirection().equals(Direction.FORWARD)) {
QUERY_MODEL.addTermToModel(mapping.getModelFieldName(), mapping.getFieldName());
} else {
QUERY_MODEL.addTermToReverseModel(mapping.getFieldName(), mapping.getModelFieldName());
}
}
reverseMap = new HashMap<>();
reverseMap.put("FIELD_1", "UUID");
reverseMap.put("FIELD_3", "R_LABEL");
reverseMap.put("FIELD_4", "FIELD_4B");
reverseMap.put("FIELD_2", "PROTOCOL");
}
use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.
the class ShardQueryConfigurationTest method testDeepCopyConstructor.
/**
* Test that for a given set of collections, stored in a ShardQueryConfiguration, will in fact be deep-copied into a new ShardQueryConfiguration object.
*/
@Test
public void testDeepCopyConstructor() {
// Instantiate a 'other' ShardQueryConfiguration
ShardQueryConfiguration other = ShardQueryConfiguration.create();
// Setup collections for deep copy
List<String> realmSuffixExclusionPatterns = Lists.newArrayList("somePattern");
SimpleDateFormat shardDateFormatter = new SimpleDateFormat("yyyyMMdd");
List<String> enricherClassNames = Lists.newArrayList("enricherClassNameA");
List<String> filterClassNames = Lists.newArrayList("filterClassNameA");
List<String> indexFilteringClassNames = Lists.newArrayList("indexFilteringClassNameA");
Set<String> nonEventKeyPrefixes = Sets.newHashSet("nonEventKeyPrefixA");
Set<String> unevaluatedFields = Sets.newHashSet("unevaluatedFieldA");
Set<String> dataTypeFilter = Sets.newHashSet("dataTypeFilterA");
IndexHole indexHole = new IndexHole(new String[] { "0", "1" }, new String[] { "2", "3" });
List<IndexHole> indexHoles = Lists.newArrayList(indexHole);
Set<String> projectFields = Sets.newHashSet("projectFieldA");
Set<String> blacklistedFields = Sets.newHashSet("blacklistedFieldA");
Set<String> indexedFields = Sets.newHashSet("indexedFieldA");
Set<String> normalizedFields = Sets.newHashSet("normalizedFieldA");
Multimap<String, Type<?>> dataTypes = HashMultimap.create();
dataTypes.put("K001", new NoOpType("V"));
Multimap<String, Type<?>> queryFieldsDatatypes = HashMultimap.create();
queryFieldsDatatypes.put("K002", new NoOpType("V"));
Multimap<String, Type<?>> normalizedFieldsDatatypes = HashMultimap.create();
normalizedFieldsDatatypes.put("K003", new NoOpType("V"));
Multimap<String, String> compositeToFieldMap = HashMultimap.create();
compositeToFieldMap.put("K004", "V");
Map<String, DiscreteIndexType<?>> fieldToDiscreteIndexType = Maps.newHashMap();
fieldToDiscreteIndexType.put("GEO", new GeometryType());
Map<String, Date> compositeTransitionDates = Maps.newHashMap();
Date transitionDate = new Date();
compositeTransitionDates.put("K005", transitionDate);
Map<String, String> compositeFieldSeparators = Maps.newHashMap();
compositeFieldSeparators.put("GEO", " ");
Set<String> queryTermFrequencyFields = Sets.newHashSet("fieldA");
Set<String> limitFields = Sets.newHashSet("limitFieldA");
Map<String, String> hierarchyFieldOptions = Maps.newHashMap();
hierarchyFieldOptions.put("K006", "V");
List<String> documentPermutations = Lists.newArrayList(DocumentPermutation.class.getName());
QueryModel queryModel = new QueryModel();
QueryImpl query = new QueryImpl();
Set<String> groupFields = Sets.newHashSet("groupFieldA");
UniqueFields uniqueFields = new UniqueFields();
uniqueFields.put("uniqueFieldA", UniqueGranularity.ALL);
List<String> contentFieldNames = Lists.newArrayList("fieldA");
Set<String> noExpansionFields = Sets.newHashSet("NoExpansionFieldA");
Set<String> disallowedRegexPatterns = Sets.newHashSet(".*", ".*?");
// Set collections on 'other' ShardQueryConfiguration
other.setRealmSuffixExclusionPatterns(realmSuffixExclusionPatterns);
other.setShardDateFormatter(shardDateFormatter);
other.setEnricherClassNames(enricherClassNames);
other.setFilterClassNames(filterClassNames);
other.setIndexFilteringClassNames(indexFilteringClassNames);
other.setNonEventKeyPrefixes(nonEventKeyPrefixes);
other.setUnevaluatedFields(unevaluatedFields);
other.setDatatypeFilter(dataTypeFilter);
other.setIndexHoles(indexHoles);
other.setProjectFields(projectFields);
other.setBlacklistedFields(blacklistedFields);
other.setIndexedFields(indexedFields);
other.setNormalizedFields(normalizedFields);
other.setDataTypes(dataTypes);
other.setQueryFieldsDatatypes(queryFieldsDatatypes);
other.setNormalizedFieldsDatatypes(normalizedFieldsDatatypes);
other.setCompositeToFieldMap(compositeToFieldMap);
other.setFieldToDiscreteIndexTypes(fieldToDiscreteIndexType);
other.setCompositeTransitionDates(compositeTransitionDates);
other.setCompositeFieldSeparators(compositeFieldSeparators);
other.setQueryTermFrequencyFields(queryTermFrequencyFields);
other.setLimitFields(limitFields);
other.setHierarchyFieldOptions(hierarchyFieldOptions);
other.setDocumentPermutations(documentPermutations);
other.setQueryModel(queryModel);
other.setQuery(query);
other.setGroupFields(groupFields);
other.setUniqueFields(uniqueFields);
other.setContentFieldNames(contentFieldNames);
other.setNoExpansionFields(noExpansionFields);
other.setDisallowedRegexPatterns(disallowedRegexPatterns);
// Copy 'other' ShardQueryConfiguration into a new config
ShardQueryConfiguration config = ShardQueryConfiguration.create(other);
// Modify original collections
realmSuffixExclusionPatterns.add("anotherPattern");
shardDateFormatter = new SimpleDateFormat("yyyyMMdd-mm:SS");
enricherClassNames.add("enricherClassNameB");
filterClassNames.add("filterClassNameB");
indexFilteringClassNames.add("indexFilteringClassNameB");
nonEventKeyPrefixes.add("nonEventKeyPrefixB");
unevaluatedFields.add("unevaluatedFieldB");
dataTypeFilter.add("dataTypeFilterB");
IndexHole otherIndexHole = new IndexHole(new String[] { "4", "5" }, new String[] { "6", "7" });
indexHoles.add(otherIndexHole);
projectFields.add("projectFieldB");
blacklistedFields.add("blacklistedFieldB");
indexedFields.add("indexedFieldB");
normalizedFields.add("normalizedFieldB");
dataTypes.put("K2", new NoOpType("V2"));
queryFieldsDatatypes.put("K", new NoOpType("V2"));
normalizedFieldsDatatypes.put("K2", new NoOpType("V2"));
compositeToFieldMap.put("K2", "V2");
queryTermFrequencyFields.add("fieldB");
limitFields.add("limitFieldB");
hierarchyFieldOptions.put("K2", "V2");
documentPermutations.add(DocumentProjection.class.getName());
queryModel.addTermToModel("aliasA", "diskNameA");
query.setId(UUID.randomUUID());
groupFields.add("groupFieldB");
uniqueFields.put("uniqueFieldB", UniqueGranularity.ALL);
contentFieldNames.add("fieldB");
disallowedRegexPatterns.add("blah");
// Assert that copied collections were deep copied and remain unchanged
Assert.assertEquals(Lists.newArrayList("somePattern"), config.getRealmSuffixExclusionPatterns());
Assert.assertEquals(new SimpleDateFormat("yyyyMMdd"), config.getShardDateFormatter());
Assert.assertEquals(Lists.newArrayList("enricherClassNameA"), config.getEnricherClassNames());
Assert.assertEquals(Lists.newArrayList("filterClassNameA"), config.getFilterClassNames());
Assert.assertEquals(Lists.newArrayList("indexFilteringClassNameA"), config.getIndexFilteringClassNames());
Assert.assertEquals(Sets.newHashSet("nonEventKeyPrefixA"), config.getNonEventKeyPrefixes());
Assert.assertEquals(Sets.newHashSet("unevaluatedFieldA"), config.getUnevaluatedFields());
Assert.assertEquals(Sets.newHashSet("dataTypeFilterA"), config.getDatatypeFilter());
IndexHole expectedIndexHole = new IndexHole(new String[] { "0", "1" }, new String[] { "2", "3" });
Assert.assertEquals(Lists.newArrayList(expectedIndexHole), config.getIndexHoles());
Assert.assertEquals(Sets.newHashSet("projectFieldA"), config.getProjectFields());
Assert.assertEquals(Sets.newHashSet("blacklistedFieldA"), config.getBlacklistedFields());
Assert.assertEquals(Sets.newHashSet("indexedFieldA"), config.getIndexedFields());
// This assert is different from the setter as setNormalizedFieldsAsDatatypes will overwrite the normalizedFields with
// a new keyset.
Assert.assertEquals(Sets.newHashSet("K003"), config.getNormalizedFields());
Multimap<String, Type<?>> expectedDataTypes = HashMultimap.create();
expectedDataTypes.put("K001", new NoOpType("V"));
Assert.assertEquals(expectedDataTypes, config.getDataTypes());
Multimap<String, Type<?>> expectedQueryFieldsDatatypes = HashMultimap.create();
expectedQueryFieldsDatatypes.put("K002", new NoOpType("V"));
Assert.assertEquals(expectedQueryFieldsDatatypes, config.getQueryFieldsDatatypes());
Multimap<String, Type<?>> expectedNormalizedFieldsDatatypes = HashMultimap.create();
expectedNormalizedFieldsDatatypes.put("K003", new NoOpType("V"));
Assert.assertEquals(expectedNormalizedFieldsDatatypes, config.getNormalizedFieldsDatatypes());
Multimap<String, String> expectedCompositeToFieldMap = ArrayListMultimap.create();
expectedCompositeToFieldMap.put("K004", "V");
Assert.assertEquals(expectedCompositeToFieldMap, config.getCompositeToFieldMap());
Map<String, DiscreteIndexType<?>> expectedFieldToDiscreteIndexType = Maps.newHashMap();
expectedFieldToDiscreteIndexType.put("GEO", new GeometryType());
Assert.assertEquals(expectedFieldToDiscreteIndexType, config.getFieldToDiscreteIndexTypes());
Map<String, Date> expectedCompositeTransitionDates = Maps.newHashMap();
expectedCompositeTransitionDates.put("K005", transitionDate);
Assert.assertEquals(expectedCompositeTransitionDates, config.getCompositeTransitionDates());
Map<String, String> expectedCompositeFieldSeparators = Maps.newHashMap();
expectedCompositeFieldSeparators.put("GEO", " ");
Assert.assertEquals(expectedCompositeFieldSeparators, config.getCompositeFieldSeparators());
Assert.assertEquals(Sets.newHashSet("fieldA"), config.getQueryTermFrequencyFields());
Assert.assertEquals(Sets.newHashSet("limitFieldA"), config.getLimitFields());
Map<String, String> expectedHierarchyFieldOptions = Maps.newHashMap();
expectedHierarchyFieldOptions.put("K006", "V");
Assert.assertEquals(expectedHierarchyFieldOptions, config.getHierarchyFieldOptions());
Assert.assertEquals(Lists.newArrayList(DocumentPermutation.class.getName()), config.getDocumentPermutations());
QueryModel expectedQueryModel = new QueryModel();
Assert.assertEquals(expectedQueryModel.getForwardQueryMapping(), config.getQueryModel().getForwardQueryMapping());
Assert.assertEquals(expectedQueryModel.getReverseQueryMapping(), config.getQueryModel().getReverseQueryMapping());
Assert.assertEquals(expectedQueryModel.getUnevaluatedFields(), config.getQueryModel().getUnevaluatedFields());
Assert.assertEquals(Sets.newHashSet(".*", ".*?"), config.getDisallowedRegexPatterns());
// Account for QueryImpl.duplicate() generating a random UUID on the duplicate
QueryImpl expectedQuery = new QueryImpl();
expectedQuery.setId(config.getQuery().getId());
Assert.assertEquals(expectedQuery, config.getQuery());
Assert.assertEquals(Sets.newHashSet("groupFieldA"), config.getGroupFields());
UniqueFields expectedUniqueFields = new UniqueFields();
expectedUniqueFields.put("uniqueFieldA", UniqueGranularity.ALL);
Assert.assertEquals(expectedUniqueFields, config.getUniqueFields());
Assert.assertEquals(Lists.newArrayList("fieldA"), config.getContentFieldNames());
Assert.assertEquals(Sets.newHashSet("NoExpansionFieldA"), config.getNoExpansionFields());
}
use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.
the class QueryModelVisitorTest method setupModel.
@Before
public void setupModel() {
model = new QueryModel();
model.addTermToModel("FOO", "BAR1");
model.addTermToModel("FOO", "BAR2");
model.addTermToModel("FIELD", "FIELD");
model.addTermToModel("CYCLIC_FIELD", "CYCLIC_FIELD");
model.addTermToModel("CYCLIC_FIELD", "CYCLIC_FIELD_");
model.addTermToModel("OUT", "IN");
model.addTermToModel("AG", "AGE");
model.addTermToModel("AG", "ETA");
model.addTermToModel("NAM", "NAME");
model.addTermToModel("NAM", "NOME");
model.addTermToModel("GEN", "GENDER");
model.addTermToModel("GEN", "GENERE");
model.addTermToReverseModel("BAR1", "FOO");
model.addTermToReverseModel("BAR2", "FOO");
model.addTermToReverseModel("FIELD", "FIELD");
model.addTermToReverseModel("CYCLIC_FIELD", "CYCLIC_FIELD");
model.addTermToReverseModel("CYCLIC_FIELD_", "CYCLIC_FIELD");
model.addTermToReverseModel("IN", "OUT");
model.addTermToReverseModel("AGE", "AG");
model.addTermToReverseModel("ETA", "AG");
model.addTermToReverseModel("NAME", "NAM");
model.addTermToReverseModel("NOME", "NAM");
model.addTermToReverseModel("GENDER", "GEN");
model.addTermToReverseModel("GENERE", "GEN");
allFields = new HashSet<>();
allFields.add("FOO");
allFields.add("9_2");
allFields.add("BAR1");
allFields.add("BAR2");
allFields.add("FIELD");
allFields.add("CYCLIC_FIELD");
allFields.add("CYCLIC_FIELD_");
allFields.add("OUT");
allFields.add("IN");
allFields.add("1BAR");
allFields.add("2BAR");
allFields.add("3BAR");
allFields.add("1FOO");
allFields.add("2FOO");
allFields.add("3FOO");
allFields.add("AG");
allFields.add("AGE");
allFields.add("ETA");
allFields.add("NAM");
allFields.add("NAME");
allFields.add("NOME");
allFields.add("GEN");
allFields.add("GENDER");
allFields.add("GENERE");
}
Aggregations