use of uk.gov.gchq.gaffer.data.elementdefinition.view.View in project Gaffer by gchq.
the class CoreKeyGroupByAggregatorIteratorTest method testAggregatingMultiplePropertySets.
private void testAggregatingMultiplePropertySets(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
String visibilityString = "public";
try {
// Create edge
final Edge edge = new Edge(TestGroups.EDGE);
edge.setSource("1");
edge.setDestination("2");
edge.setDirected(true);
edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge.putProperty(AccumuloPropertyNames.COUNT, 1);
//THIS EDGE WILL BE REDUCED MEANING ITS CQ (columnQualifier) will only occur once because its key is equal.
final Edge edge2 = new Edge(TestGroups.EDGE);
edge2.setSource("1");
edge2.setDestination("2");
edge2.setDirected(true);
edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge2.putProperty(AccumuloPropertyNames.COUNT, 2);
final Edge edge3 = new Edge(TestGroups.EDGE);
edge3.setSource("1");
edge3.setDestination("2");
edge3.setDirected(true);
edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge3.putProperty(AccumuloPropertyNames.COUNT, 10);
// Accumulo key
final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
// Accumulo values
final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge.getProperties());
final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge2.getProperties());
final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge3.getProperties());
// Create mutation
final Mutation m1 = new Mutation(key.getRow());
m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
final Mutation m2 = new Mutation(key.getRow());
m2.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value2);
final Mutation m3 = new Mutation(key.getRow());
m3.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value3);
final Mutation m4 = new Mutation(key2.getRow());
m4.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value1);
final Mutation m5 = new Mutation(key.getRow());
m5.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value1);
// Write mutation
final BatchWriterConfig writerConfig = new BatchWriterConfig();
writerConfig.setMaxMemory(1000000L);
writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
writerConfig.setMaxWriteThreads(1);
final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
writer.addMutation(m1);
writer.addMutation(m2);
writer.addMutation(m3);
writer.addMutation(m4);
writer.addMutation(m5);
writer.close();
// Read data back and check we get one merged element
final Authorizations authorizations = new Authorizations(visibilityString);
final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().schema(store.getSchema()).view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
scanner.addScanIterator(iteratorSetting);
final Iterator<Entry<Key, Value>> it = scanner.iterator();
final Entry<Key, Value> entry = it.next();
final Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
final Edge expectedEdge = new Edge(TestGroups.EDGE);
expectedEdge.setSource("1");
expectedEdge.setDestination("2");
expectedEdge.setDirected(true);
expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 9);
expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 15);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_1, 0);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_2, 0);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_3, 0);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_4, 0);
assertEquals(expectedEdge, readEdge);
assertEquals(9, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(15, readEdge.getProperty(AccumuloPropertyNames.COUNT));
// Check no more entries
if (it.hasNext()) {
fail("Additional row found.");
}
} catch (AccumuloException | TableNotFoundException e) {
fail(this.getClass().getSimpleName() + " failed with exception: " + e);
}
}
use of uk.gov.gchq.gaffer.data.elementdefinition.view.View in project Gaffer by gchq.
the class AccumuloStoreTest method testAbleToInsertAndRetrieveEntityQueryingEqualAndRelated.
public void testAbleToInsertAndRetrieveEntityQueryingEqualAndRelated(AccumuloStore store) throws OperationException {
final List<Element> elements = new ArrayList<>();
final Entity e = new Entity(TestGroups.ENTITY, "1");
e.putProperty(TestPropertyNames.PROP_1, 1);
e.putProperty(TestPropertyNames.PROP_2, 2);
e.putProperty(TestPropertyNames.PROP_3, 3);
e.putProperty(TestPropertyNames.PROP_4, 4);
e.putProperty(TestPropertyNames.COUNT, 1);
final User user = new User();
elements.add(e);
final AddElements add = new AddElements.Builder().elements(elements).build();
store.execute(add, user);
final EntitySeed entitySeed1 = new EntitySeed("1");
final GetElements<EntitySeed, Element> getBySeed = new GetElements.Builder<EntitySeed, Element>().view(new View.Builder().entity(TestGroups.ENTITY).build()).addSeed(entitySeed1).build();
final CloseableIterable<Element> results = store.execute(getBySeed, user);
assertEquals(1, Iterables.size(results));
assertThat(results, IsCollectionContaining.hasItem(e));
final GetElements<EntitySeed, Element> getRelated = new GetElements.Builder<EntitySeed, Element>().view(new View.Builder().entity(TestGroups.ENTITY).build()).addSeed(entitySeed1).build();
CloseableIterable<Element> relatedResults = store.execute(getRelated, user);
assertEquals(1, Iterables.size(relatedResults));
assertThat(relatedResults, IsCollectionContaining.hasItem(e));
final GetElements<EntitySeed, Element> getRelatedWithPostAggregationFilter = new GetElements.Builder<EntitySeed, Element>().view(new View.Builder().entity(TestGroups.ENTITY, new ViewElementDefinition.Builder().preAggregationFilter(new ElementFilter.Builder().select(TestPropertyNames.PROP_1).execute(new IsMoreThan(0)).build()).postAggregationFilter(new ElementFilter.Builder().select(TestPropertyNames.COUNT).execute(new IsMoreThan(6)).build()).build()).build()).addSeed(entitySeed1).build();
relatedResults = store.execute(getRelatedWithPostAggregationFilter, user);
assertEquals(0, Iterables.size(relatedResults));
}
use of uk.gov.gchq.gaffer.data.elementdefinition.view.View in project Gaffer by gchq.
the class InputFormatTest method shouldReturnCorrectDataToMapReduceJob.
@Test
public void shouldReturnCorrectDataToMapReduceJob() throws Exception {
final View view = new View.Builder().build();
final Set<String> expectedResults = new HashSet<>();
for (final Element element : DATA) {
expectedResults.add(element.toString());
}
shouldReturnCorrectDataToMapReduceJob(getSchema(), KeyPackage.BYTE_ENTITY_KEY_PACKAGE, DATA, view, new User(), "instance1", expectedResults);
shouldReturnCorrectDataToMapReduceJob(getSchema(), KeyPackage.CLASSIC_KEY_PACKAGE, DATA, view, new User(), "instance2", expectedResults);
}
use of uk.gov.gchq.gaffer.data.elementdefinition.view.View in project Gaffer by gchq.
the class InputFormatTest method shouldReturnCorrectDataToMapReduceJobRespectingAuthorizations.
@Test
public void shouldReturnCorrectDataToMapReduceJobRespectingAuthorizations() throws Exception {
final Schema schema = getSchemaWithVisibilities();
final View view = new View.Builder().build();
final Set<String> expectedResultsPublicNotPrivate = new HashSet<>();
final Set<String> expectedResultsPrivate = new HashSet<>();
for (final Element element : DATA_WITH_VISIBILITIES) {
expectedResultsPrivate.add(element.toString());
if (element.getProperty("visibility").equals("public")) {
expectedResultsPublicNotPrivate.add(element.toString());
}
}
final Set<String> privateAuth = new HashSet<>();
privateAuth.add("public");
privateAuth.add("private");
final Set<String> publicNotPrivate = new HashSet<>();
publicNotPrivate.add("public");
final User userWithPrivate = new User("user1", privateAuth);
final User userWithPublicNotPrivate = new User("user1", publicNotPrivate);
shouldReturnCorrectDataToMapReduceJob(schema, KeyPackage.BYTE_ENTITY_KEY_PACKAGE, DATA_WITH_VISIBILITIES, view, userWithPublicNotPrivate, "instance5", expectedResultsPublicNotPrivate);
shouldReturnCorrectDataToMapReduceJob(schema, KeyPackage.BYTE_ENTITY_KEY_PACKAGE, DATA_WITH_VISIBILITIES, view, userWithPrivate, "instance6", expectedResultsPrivate);
shouldReturnCorrectDataToMapReduceJob(schema, KeyPackage.CLASSIC_KEY_PACKAGE, DATA_WITH_VISIBILITIES, view, userWithPublicNotPrivate, "instance7", expectedResultsPublicNotPrivate);
shouldReturnCorrectDataToMapReduceJob(schema, KeyPackage.CLASSIC_KEY_PACKAGE, DATA_WITH_VISIBILITIES, view, userWithPrivate, "instance8", expectedResultsPrivate);
}
use of uk.gov.gchq.gaffer.data.elementdefinition.view.View in project Gaffer by gchq.
the class InputFormatTest method shouldReturnCorrectDataToMapReduceJobWithView.
@Test
public void shouldReturnCorrectDataToMapReduceJobWithView() throws Exception {
final Schema schema = getSchema();
final View view = new View.Builder().edge(TestGroups.EDGE).build();
final Set<String> expectedResults = new HashSet<>();
for (final Element element : DATA) {
if (element.getGroup().equals(TestGroups.EDGE)) {
expectedResults.add(element.toString());
}
}
shouldReturnCorrectDataToMapReduceJob(schema, KeyPackage.BYTE_ENTITY_KEY_PACKAGE, DATA, view, new User(), "instance3", expectedResults);
shouldReturnCorrectDataToMapReduceJob(schema, KeyPackage.CLASSIC_KEY_PACKAGE, DATA, view, new User(), "instance4", expectedResults);
}
Aggregations