Example 11 with IteratorSetting
use of org.apache.accumulo.core.client.IteratorSetting in project presto by prestodb.
the class TestIndexer method testMutationIndexWithVisibilities.
@Test
public void testMutationIndexWithVisibilities() throws Exception {
Instance inst = new MockInstance();
Connector conn = inst.getConnector("root", new PasswordToken(""));
conn.tableOperations().create(table.getFullTableName());
conn.tableOperations().create(table.getIndexTableName());
conn.tableOperations().create(table.getMetricsTableName());
for (IteratorSetting s : Indexer.getMetricIterators(table)) {
conn.tableOperations().attachIterator(table.getMetricsTableName(), s);
}
Indexer indexer = new Indexer(conn, new Authorizations(), table, new BatchWriterConfig());
indexer.index(m1);
indexer.index(m1v);
indexer.flush();
Scanner scan = conn.createScanner(table.getIndexTableName(), new Authorizations("private", "moreprivate"));
scan.setRange(new Range());
Iterator<Entry<Key, Value>> iter = scan.iterator();
assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", "");
assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", "private", "");
assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", "");
assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", "moreprivate", "");
assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", "");
assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", "private", "");
assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", "");
assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", "moreprivate", "");
assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", "");
assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", "moreprivate", "");
assertFalse(iter.hasNext());
scan.close();
scan = conn.createScanner(table.getMetricsTableName(), new Authorizations("private", "moreprivate"));
scan.setRange(new Range());
iter = scan.iterator();
assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "1");
assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "private", "1");
assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___card___", "2");
assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___first_row___", "row1");
assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___last_row___", "row1");
assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "1");
assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "moreprivate", "1");
assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "1");
assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "private", "1");
assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "1");
assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "moreprivate", "1");
assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "1");
assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "moreprivate", "1");
assertFalse(iter.hasNext());
scan.close();
indexer.index(m2);
indexer.index(m2v);
indexer.close();
scan = conn.createScanner(table.getIndexTableName(), new Authorizations("private", "moreprivate"));
scan.setRange(new Range());
iter = scan.iterator();
assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", "");
assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", "private", "");
assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row2", "");
assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row2", "private", "");
assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", "");
assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", "moreprivate", "");
assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row2", "");
assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row2", "moreprivate", "");
assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", "");
assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", "private", "");
assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "row2", "");
assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "row2", "moreprivate", "");
assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", "");
assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", "moreprivate", "");
assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", "");
assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", "moreprivate", "");
assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row2", "");
assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row2", "moreprivate", "");
assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "row2", "");
assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "row2", "moreprivate", "");
assertFalse(iter.hasNext());
scan.close();
scan = conn.createScanner(table.getMetricsTableName(), new Authorizations("private", "moreprivate"));
scan.setRange(new Range());
iter = scan.iterator();
assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "2");
assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "private", "2");
assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___card___", "4");
assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___first_row___", "row1");
assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___last_row___", "row2");
assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "2");
assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "moreprivate", "2");
assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "1");
assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "private", "1");
assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "___card___", "1");
assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "___card___", "moreprivate", "1");
assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "1");
assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "moreprivate", "1");
assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "2");
assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "moreprivate", "2");
assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "___card___", "1");
assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "___card___", "moreprivate", "1");
assertFalse(iter.hasNext());
scan.close();
}
Also used :
Connector(org.apache.accumulo.core.client.Connector)
Scanner(org.apache.accumulo.core.client.Scanner)
PasswordToken(org.apache.accumulo.core.client.security.tokens.PasswordToken)
Authorizations(org.apache.accumulo.core.security.Authorizations)
Entry(java.util.Map.Entry)
IteratorSetting(org.apache.accumulo.core.client.IteratorSetting)
MockInstance(org.apache.accumulo.core.client.mock.MockInstance)
Instance(org.apache.accumulo.core.client.Instance)
MockInstance(org.apache.accumulo.core.client.mock.MockInstance)
BatchWriterConfig(org.apache.accumulo.core.client.BatchWriterConfig)
Range(org.apache.accumulo.core.data.Range)
Test(org.testng.annotations.Test)
Example 12 with IteratorSetting
use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.
the class CoreKeyGroupByAggregatorIteratorTest method shouldPartiallyAggregateColumnQualifierOverCQ1GroupBy.
public void shouldPartiallyAggregateColumnQualifierOverCQ1GroupBy(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
final String visibilityString = "public";
try {
// Create edge
final Edge edge = new Edge(TestGroups.EDGE);
edge.setSource("1");
edge.setDestination("2");
edge.setDirected(true);
edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge.putProperty(AccumuloPropertyNames.COUNT, 1);
final Edge edge2 = new Edge(TestGroups.EDGE);
edge2.setSource("1");
edge2.setDestination("2");
edge2.setDirected(true);
edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 2);
edge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge2.putProperty(AccumuloPropertyNames.COUNT, 1);
final Edge edge3 = new Edge(TestGroups.EDGE);
edge3.setSource("1");
edge3.setDestination("2");
edge3.setDirected(true);
edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 3);
edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge3.putProperty(AccumuloPropertyNames.COUNT, 1);
//THIS EDGE WILL BE REDUCED MEANING ITS CQ (columnQualifier) will only occur once because its key is equal.
final Edge edge4 = new Edge(TestGroups.EDGE);
edge4.setSource("1");
edge4.setDestination("2");
edge4.setDirected(true);
edge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
edge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 4);
edge4.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge4.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge4.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge4.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge4.putProperty(AccumuloPropertyNames.COUNT, 2);
final Edge edge5 = new Edge(TestGroups.EDGE);
edge5.setSource("1");
edge5.setDestination("2");
edge5.setDirected(true);
edge5.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 3);
edge5.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 5);
edge5.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge5.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge5.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge5.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge5.putProperty(AccumuloPropertyNames.COUNT, 10);
final Edge edge6 = new Edge(TestGroups.EDGE);
edge6.setSource("1");
edge6.setDestination("2");
edge6.setDirected(true);
edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 4);
edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 6);
edge6.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge6.putProperty(AccumuloPropertyNames.COUNT, 5);
// Accumulo key
final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
final Key key4 = elementConverter.getKeysFromEdge(edge4).getFirst();
final Key key5 = elementConverter.getKeysFromEdge(edge5).getFirst();
final Key key6 = elementConverter.getKeysFromEdge(edge6).getFirst();
// Accumulo values
final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge.getProperties());
final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge2.getProperties());
final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge3.getProperties());
final Value value4 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge4.getProperties());
final Value value5 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge5.getProperties());
final Value value6 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge6.getProperties());
// Create mutation
final Mutation m1 = new Mutation(key.getRow());
m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
final Mutation m2 = new Mutation(key2.getRow());
m2.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value2);
final Mutation m3 = new Mutation(key.getRow());
m3.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value3);
final Mutation m4 = new Mutation(key.getRow());
m4.put(key4.getColumnFamily(), key4.getColumnQualifier(), new ColumnVisibility(key4.getColumnVisibility()), key4.getTimestamp(), value4);
final Mutation m5 = new Mutation(key.getRow());
m5.put(key5.getColumnFamily(), key5.getColumnQualifier(), new ColumnVisibility(key5.getColumnVisibility()), key5.getTimestamp(), value5);
final Mutation m6 = new Mutation(key.getRow());
m6.put(key6.getColumnFamily(), key6.getColumnQualifier(), new ColumnVisibility(key6.getColumnVisibility()), key6.getTimestamp(), value6);
// Write mutation
final BatchWriterConfig writerConfig = new BatchWriterConfig();
writerConfig.setMaxMemory(1000000L);
writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
writerConfig.setMaxWriteThreads(1);
final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
writer.addMutation(m1);
writer.addMutation(m2);
writer.addMutation(m3);
writer.addMutation(m4);
writer.addMutation(m5);
writer.addMutation(m6);
writer.close();
Edge expectedEdge1 = new Edge(TestGroups.EDGE);
expectedEdge1.setSource("1");
expectedEdge1.setDestination("2");
expectedEdge1.setDirected(true);
expectedEdge1.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
expectedEdge1.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 6);
expectedEdge1.putProperty(AccumuloPropertyNames.COUNT, 3);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_1, 0);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_2, 0);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_3, 0);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_4, 0);
Edge expectedEdge2 = new Edge(TestGroups.EDGE);
expectedEdge2.setSource("1");
expectedEdge2.setDestination("2");
expectedEdge2.setDirected(true);
expectedEdge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
expectedEdge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 4);
expectedEdge2.putProperty(AccumuloPropertyNames.COUNT, 2);
expectedEdge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
expectedEdge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
expectedEdge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
expectedEdge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
Edge expectedEdge3 = new Edge(TestGroups.EDGE);
expectedEdge3.setSource("1");
expectedEdge3.setDestination("2");
expectedEdge3.setDirected(true);
expectedEdge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 3);
expectedEdge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 5);
expectedEdge3.putProperty(AccumuloPropertyNames.COUNT, 10);
expectedEdge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
expectedEdge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
expectedEdge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
expectedEdge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
Edge expectedEdge4 = new Edge(TestGroups.EDGE);
expectedEdge4.setSource("1");
expectedEdge4.setDestination("2");
expectedEdge4.setDirected(true);
expectedEdge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 4);
expectedEdge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 6);
expectedEdge4.putProperty(AccumuloPropertyNames.COUNT, 5);
expectedEdge4.putProperty(AccumuloPropertyNames.PROP_1, 0);
expectedEdge4.putProperty(AccumuloPropertyNames.PROP_2, 0);
expectedEdge4.putProperty(AccumuloPropertyNames.PROP_3, 0);
expectedEdge4.putProperty(AccumuloPropertyNames.PROP_4, 0);
// Read data back and check we get one merged element
final Authorizations authorizations = new Authorizations(visibilityString);
final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy(AccumuloPropertyNames.COLUMN_QUALIFIER).build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
scanner.addScanIterator(iteratorSetting);
final Iterator<Entry<Key, Value>> it = scanner.iterator();
Entry<Key, Value> entry = it.next();
Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
assertEquals(expectedEdge1, readEdge);
assertEquals(1, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(6, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
assertEquals(3, readEdge.getProperty(AccumuloPropertyNames.COUNT));
entry = it.next();
readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
assertEquals(expectedEdge2, readEdge);
assertEquals(2, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(4, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
assertEquals(2, readEdge.getProperty(AccumuloPropertyNames.COUNT));
entry = it.next();
readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
assertEquals(expectedEdge3, readEdge);
assertEquals(3, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(5, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
assertEquals(10, readEdge.getProperty(AccumuloPropertyNames.COUNT));
entry = it.next();
readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
assertEquals(expectedEdge4, readEdge);
assertEquals(4, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(6, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
assertEquals(5, readEdge.getProperty(AccumuloPropertyNames.COUNT));
if (it.hasNext()) {
fail("Additional row found.");
}
} catch (AccumuloException | TableNotFoundException e) {
fail(this.getClass().getSimpleName() + " failed with exception: " + e);
}
}
Also used :
Scanner(org.apache.accumulo.core.client.Scanner)
AccumuloException(org.apache.accumulo.core.client.AccumuloException)
Authorizations(org.apache.accumulo.core.security.Authorizations)
IteratorSettingBuilder(uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder)
Element(uk.gov.gchq.gaffer.data.element.Element)
IteratorSettingBuilder(uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder)
TableNotFoundException(org.apache.accumulo.core.client.TableNotFoundException)
Entry(java.util.Map.Entry)
IteratorSetting(org.apache.accumulo.core.client.IteratorSetting)
Value(org.apache.accumulo.core.data.Value)
BatchWriterConfig(org.apache.accumulo.core.client.BatchWriterConfig)
Mutation(org.apache.accumulo.core.data.Mutation)
ColumnVisibility(org.apache.accumulo.core.security.ColumnVisibility)
BatchWriter(org.apache.accumulo.core.client.BatchWriter)
Edge(uk.gov.gchq.gaffer.data.element.Edge)
Key(org.apache.accumulo.core.data.Key)
Example 13 with IteratorSetting
use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.
the class CoreKeyGroupByAggregatorIteratorTest method shouldAggregatePropertiesOnlyWhenGroupByIsSetToCQ1CQ2.
public void shouldAggregatePropertiesOnlyWhenGroupByIsSetToCQ1CQ2(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
final String visibilityString = "public";
try {
// Create edge
final Edge edge = new Edge(TestGroups.EDGE);
edge.setSource("1");
edge.setDestination("2");
edge.setDirected(true);
edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge.putProperty(AccumuloPropertyNames.COUNT, 1);
final Edge edge2 = new Edge(TestGroups.EDGE);
edge2.setSource("1");
edge2.setDestination("2");
edge2.setDirected(true);
edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
edge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge2.putProperty(AccumuloPropertyNames.COUNT, 1);
final Edge edge3 = new Edge(TestGroups.EDGE);
edge3.setSource("1");
edge3.setDestination("2");
edge3.setDirected(true);
edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge3.putProperty(AccumuloPropertyNames.COUNT, 1);
//THIS EDGE WILL BE REDUCED MEANING ITS CQ (columnQualifier) will only occur once because its key is equal.
final Edge edge4 = new Edge(TestGroups.EDGE);
edge4.setSource("1");
edge4.setDestination("2");
edge4.setDirected(true);
edge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 4);
edge4.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge4.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge4.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge4.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge4.putProperty(AccumuloPropertyNames.COUNT, 2);
final Edge edge5 = new Edge(TestGroups.EDGE);
edge5.setSource("1");
edge5.setDestination("2");
edge5.setDirected(true);
edge5.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 3);
edge5.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 5);
edge5.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge5.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge5.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge5.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge5.putProperty(AccumuloPropertyNames.COUNT, 10);
final Edge edge6 = new Edge(TestGroups.EDGE);
edge6.setSource("1");
edge6.setDestination("2");
edge6.setDirected(true);
edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 3);
edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 5);
edge6.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge6.putProperty(AccumuloPropertyNames.COUNT, 5);
// Accumulo key
final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
final Key key4 = elementConverter.getKeysFromEdge(edge4).getFirst();
final Key key5 = elementConverter.getKeysFromEdge(edge5).getFirst();
final Key key6 = elementConverter.getKeysFromEdge(edge6).getFirst();
// Accumulo values
final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge.getProperties());
final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge2.getProperties());
final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge3.getProperties());
final Value value4 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge4.getProperties());
final Value value5 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge5.getProperties());
final Value value6 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge6.getProperties());
// Create mutation
final Mutation m1 = new Mutation(key.getRow());
m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
final Mutation m2 = new Mutation(key2.getRow());
m2.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value2);
final Mutation m3 = new Mutation(key.getRow());
m3.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value3);
final Mutation m4 = new Mutation(key.getRow());
m4.put(key4.getColumnFamily(), key4.getColumnQualifier(), new ColumnVisibility(key4.getColumnVisibility()), key4.getTimestamp(), value4);
final Mutation m5 = new Mutation(key.getRow());
m5.put(key5.getColumnFamily(), key5.getColumnQualifier(), new ColumnVisibility(key5.getColumnVisibility()), key5.getTimestamp(), value5);
final Mutation m6 = new Mutation(key.getRow());
m6.put(key6.getColumnFamily(), key6.getColumnQualifier(), new ColumnVisibility(key6.getColumnVisibility()), key6.getTimestamp(), value6);
// Write mutation
final BatchWriterConfig writerConfig = new BatchWriterConfig();
writerConfig.setMaxMemory(1000000L);
writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
writerConfig.setMaxWriteThreads(1);
final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
writer.addMutation(m1);
writer.addMutation(m2);
writer.addMutation(m3);
writer.addMutation(m4);
writer.addMutation(m5);
writer.addMutation(m6);
writer.close();
Edge expectedEdge1 = new Edge(TestGroups.EDGE);
expectedEdge1.setSource("1");
expectedEdge1.setDestination("2");
expectedEdge1.setDirected(true);
expectedEdge1.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
expectedEdge1.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
expectedEdge1.putProperty(AccumuloPropertyNames.COUNT, 3);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_1, 0);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_2, 0);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_3, 0);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_4, 0);
Edge expectedEdge2 = new Edge(TestGroups.EDGE);
expectedEdge2.setSource("1");
expectedEdge2.setDestination("2");
expectedEdge2.setDirected(true);
expectedEdge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
expectedEdge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 4);
expectedEdge2.putProperty(AccumuloPropertyNames.COUNT, 2);
expectedEdge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
expectedEdge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
expectedEdge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
expectedEdge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
Edge expectedEdge3 = new Edge(TestGroups.EDGE);
expectedEdge3.setSource("1");
expectedEdge3.setDestination("2");
expectedEdge3.setDirected(true);
expectedEdge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 3);
expectedEdge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 5);
expectedEdge3.putProperty(AccumuloPropertyNames.COUNT, 15);
expectedEdge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
expectedEdge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
expectedEdge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
expectedEdge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
// Read data back and check we get one merged element
final Authorizations authorizations = new Authorizations(visibilityString);
final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy(AccumuloPropertyNames.COLUMN_QUALIFIER, AccumuloPropertyNames.COLUMN_QUALIFIER_2).build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
scanner.addScanIterator(iteratorSetting);
final Iterator<Entry<Key, Value>> it = scanner.iterator();
Entry<Key, Value> entry = it.next();
Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
assertEquals(expectedEdge1, readEdge);
assertEquals(1, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(1, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
assertEquals(3, readEdge.getProperty(AccumuloPropertyNames.COUNT));
entry = it.next();
readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
assertEquals(expectedEdge2, readEdge);
assertEquals(1, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(4, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
assertEquals(2, readEdge.getProperty(AccumuloPropertyNames.COUNT));
entry = it.next();
readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
assertEquals(expectedEdge3, readEdge);
assertEquals(3, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(5, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
assertEquals(15, readEdge.getProperty(AccumuloPropertyNames.COUNT));
if (it.hasNext()) {
fail("Additional row found.");
}
} catch (AccumuloException | TableNotFoundException e) {
fail(this.getClass().getSimpleName() + " failed with exception: " + e);
}
}
Also used :
Scanner(org.apache.accumulo.core.client.Scanner)
AccumuloException(org.apache.accumulo.core.client.AccumuloException)
Authorizations(org.apache.accumulo.core.security.Authorizations)
IteratorSettingBuilder(uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder)
Element(uk.gov.gchq.gaffer.data.element.Element)
IteratorSettingBuilder(uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder)
TableNotFoundException(org.apache.accumulo.core.client.TableNotFoundException)
Entry(java.util.Map.Entry)
IteratorSetting(org.apache.accumulo.core.client.IteratorSetting)
Value(org.apache.accumulo.core.data.Value)
BatchWriterConfig(org.apache.accumulo.core.client.BatchWriterConfig)
Mutation(org.apache.accumulo.core.data.Mutation)
ColumnVisibility(org.apache.accumulo.core.security.ColumnVisibility)
BatchWriter(org.apache.accumulo.core.client.BatchWriter)
Edge(uk.gov.gchq.gaffer.data.element.Edge)
Key(org.apache.accumulo.core.data.Key)
Example 14 with IteratorSetting
use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.
the class CoreKeyGroupByAggregatorIteratorTest method testAggregatingMultiplePropertySets.
private void testAggregatingMultiplePropertySets(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
String visibilityString = "public";
try {
// Create edge
final Edge edge = new Edge(TestGroups.EDGE);
edge.setSource("1");
edge.setDestination("2");
edge.setDirected(true);
edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge.putProperty(AccumuloPropertyNames.COUNT, 1);
//THIS EDGE WILL BE REDUCED MEANING ITS CQ (columnQualifier) will only occur once because its key is equal.
final Edge edge2 = new Edge(TestGroups.EDGE);
edge2.setSource("1");
edge2.setDestination("2");
edge2.setDirected(true);
edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge2.putProperty(AccumuloPropertyNames.COUNT, 2);
final Edge edge3 = new Edge(TestGroups.EDGE);
edge3.setSource("1");
edge3.setDestination("2");
edge3.setDirected(true);
edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge3.putProperty(AccumuloPropertyNames.COUNT, 10);
// Accumulo key
final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
// Accumulo values
final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge.getProperties());
final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge2.getProperties());
final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge3.getProperties());
// Create mutation
final Mutation m1 = new Mutation(key.getRow());
m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
final Mutation m2 = new Mutation(key.getRow());
m2.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value2);
final Mutation m3 = new Mutation(key.getRow());
m3.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value3);
final Mutation m4 = new Mutation(key2.getRow());
m4.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value1);
final Mutation m5 = new Mutation(key.getRow());
m5.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value1);
// Write mutation
final BatchWriterConfig writerConfig = new BatchWriterConfig();
writerConfig.setMaxMemory(1000000L);
writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
writerConfig.setMaxWriteThreads(1);
final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
writer.addMutation(m1);
writer.addMutation(m2);
writer.addMutation(m3);
writer.addMutation(m4);
writer.addMutation(m5);
writer.close();
// Read data back and check we get one merged element
final Authorizations authorizations = new Authorizations(visibilityString);
final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().schema(store.getSchema()).view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
scanner.addScanIterator(iteratorSetting);
final Iterator<Entry<Key, Value>> it = scanner.iterator();
final Entry<Key, Value> entry = it.next();
final Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
final Edge expectedEdge = new Edge(TestGroups.EDGE);
expectedEdge.setSource("1");
expectedEdge.setDestination("2");
expectedEdge.setDirected(true);
expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 9);
expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 15);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_1, 0);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_2, 0);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_3, 0);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_4, 0);
assertEquals(expectedEdge, readEdge);
assertEquals(9, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(15, readEdge.getProperty(AccumuloPropertyNames.COUNT));
// Check no more entries
if (it.hasNext()) {
fail("Additional row found.");
}
} catch (AccumuloException | TableNotFoundException e) {
fail(this.getClass().getSimpleName() + " failed with exception: " + e);
}
}
Also used :
Scanner(org.apache.accumulo.core.client.Scanner)
AccumuloException(org.apache.accumulo.core.client.AccumuloException)
Authorizations(org.apache.accumulo.core.security.Authorizations)
Element(uk.gov.gchq.gaffer.data.element.Element)
ViewElementDefinition(uk.gov.gchq.gaffer.data.elementdefinition.view.ViewElementDefinition)
IteratorSettingBuilder(uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder)
View(uk.gov.gchq.gaffer.data.elementdefinition.view.View)
TableNotFoundException(org.apache.accumulo.core.client.TableNotFoundException)
Entry(java.util.Map.Entry)
IteratorSetting(org.apache.accumulo.core.client.IteratorSetting)
Value(org.apache.accumulo.core.data.Value)
BatchWriterConfig(org.apache.accumulo.core.client.BatchWriterConfig)
Mutation(org.apache.accumulo.core.data.Mutation)
ColumnVisibility(org.apache.accumulo.core.security.ColumnVisibility)
BatchWriter(org.apache.accumulo.core.client.BatchWriter)
Edge(uk.gov.gchq.gaffer.data.element.Edge)
Key(org.apache.accumulo.core.data.Key)
Example 15 with IteratorSetting
use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.
the class CoreKeyGroupByAggregatorIteratorTest method shouldAggregateEverythingWhenGroupByIsSetToBlank.
public void shouldAggregateEverythingWhenGroupByIsSetToBlank(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
final String visibilityString = "public";
try {
// Create edge
final Edge edge = new Edge(TestGroups.EDGE);
edge.setSource("1");
edge.setDestination("2");
edge.setDirected(true);
edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge.putProperty(AccumuloPropertyNames.COUNT, 1);
final Edge edge2 = new Edge(TestGroups.EDGE);
edge2.setSource("1");
edge2.setDestination("2");
edge2.setDirected(true);
edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
edge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge2.putProperty(AccumuloPropertyNames.COUNT, 1);
final Edge edge3 = new Edge(TestGroups.EDGE);
edge3.setSource("1");
edge3.setDestination("2");
edge3.setDirected(true);
edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge3.putProperty(AccumuloPropertyNames.COUNT, 1);
//THIS EDGE WILL BE REDUCED MEANING ITS CQ (columnQualifier) will only occur once because its key is equal.
final Edge edge4 = new Edge(TestGroups.EDGE);
edge4.setSource("1");
edge4.setDestination("2");
edge4.setDirected(true);
edge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 4);
edge4.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge4.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge4.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge4.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge4.putProperty(AccumuloPropertyNames.COUNT, 2);
final Edge edge5 = new Edge(TestGroups.EDGE);
edge5.setSource("1");
edge5.setDestination("2");
edge5.setDirected(true);
edge5.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 3);
edge5.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 5);
edge5.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge5.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge5.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge5.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge5.putProperty(AccumuloPropertyNames.COUNT, 10);
final Edge edge6 = new Edge(TestGroups.EDGE);
edge6.setSource("1");
edge6.setDestination("2");
edge6.setDirected(true);
edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 3);
edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 5);
edge6.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge6.putProperty(AccumuloPropertyNames.COUNT, 5);
// Accumulo key
final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
final Key key4 = elementConverter.getKeysFromEdge(edge4).getFirst();
final Key key5 = elementConverter.getKeysFromEdge(edge5).getFirst();
final Key key6 = elementConverter.getKeysFromEdge(edge6).getFirst();
// Accumulo values
final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge.getProperties());
final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge2.getProperties());
final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge3.getProperties());
final Value value4 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge4.getProperties());
final Value value5 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge5.getProperties());
final Value value6 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge6.getProperties());
// Create mutation
final Mutation m1 = new Mutation(key.getRow());
m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
final Mutation m2 = new Mutation(key2.getRow());
m2.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value2);
final Mutation m3 = new Mutation(key.getRow());
m3.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value3);
final Mutation m4 = new Mutation(key.getRow());
m4.put(key4.getColumnFamily(), key4.getColumnQualifier(), new ColumnVisibility(key4.getColumnVisibility()), key4.getTimestamp(), value4);
final Mutation m5 = new Mutation(key.getRow());
m5.put(key5.getColumnFamily(), key5.getColumnQualifier(), new ColumnVisibility(key5.getColumnVisibility()), key5.getTimestamp(), value5);
final Mutation m6 = new Mutation(key.getRow());
m6.put(key6.getColumnFamily(), key6.getColumnQualifier(), new ColumnVisibility(key6.getColumnVisibility()), key6.getTimestamp(), value6);
// Write mutation
final BatchWriterConfig writerConfig = new BatchWriterConfig();
writerConfig.setMaxMemory(1000000L);
writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
writerConfig.setMaxWriteThreads(1);
final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
writer.addMutation(m1);
writer.addMutation(m2);
writer.addMutation(m3);
writer.addMutation(m4);
writer.addMutation(m5);
writer.addMutation(m6);
writer.close();
Edge expectedEdge1 = new Edge(TestGroups.EDGE);
expectedEdge1.setSource("1");
expectedEdge1.setDestination("2");
expectedEdge1.setDirected(true);
expectedEdge1.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 5);
expectedEdge1.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 10);
expectedEdge1.putProperty(AccumuloPropertyNames.COUNT, 20);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_1, 0);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_2, 0);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_3, 0);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_4, 0);
// Read data back and check we get one merged element
final Authorizations authorizations = new Authorizations(visibilityString);
final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
scanner.addScanIterator(iteratorSetting);
final Iterator<Entry<Key, Value>> it = scanner.iterator();
Entry<Key, Value> entry = it.next();
Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
assertEquals(expectedEdge1, readEdge);
assertEquals(5, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(10, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
assertEquals(20, readEdge.getProperty(AccumuloPropertyNames.COUNT));
if (it.hasNext()) {
fail("Additional row found.");
}
} catch (AccumuloException | TableNotFoundException e) {
fail(this.getClass().getSimpleName() + " failed with exception: " + e);
}
}
Also used :
Scanner(org.apache.accumulo.core.client.Scanner)
AccumuloException(org.apache.accumulo.core.client.AccumuloException)
Authorizations(org.apache.accumulo.core.security.Authorizations)
IteratorSettingBuilder(uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder)
Element(uk.gov.gchq.gaffer.data.element.Element)
IteratorSettingBuilder(uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder)
TableNotFoundException(org.apache.accumulo.core.client.TableNotFoundException)
Entry(java.util.Map.Entry)
IteratorSetting(org.apache.accumulo.core.client.IteratorSetting)
Value(org.apache.accumulo.core.data.Value)
BatchWriterConfig(org.apache.accumulo.core.client.BatchWriterConfig)
Mutation(org.apache.accumulo.core.data.Mutation)
ColumnVisibility(org.apache.accumulo.core.security.ColumnVisibility)
BatchWriter(org.apache.accumulo.core.client.BatchWriter)
Edge(uk.gov.gchq.gaffer.data.element.Edge)
Key(org.apache.accumulo.core.data.Key)
Aggregations
IteratorSetting (org.apache.accumulo.core.client.IteratorSetting)35
Test (org.junit.Test)16
Scanner (org.apache.accumulo.core.client.Scanner)13
Authorizations (org.apache.accumulo.core.security.Authorizations)13
Key (org.apache.accumulo.core.data.Key)11
Value (org.apache.accumulo.core.data.Value)11
PasswordToken (org.apache.accumulo.core.client.security.tokens.PasswordToken)10
ColumnMapper (org.apache.hadoop.hive.accumulo.columns.ColumnMapper)10
Entry (java.util.Map.Entry)9
BatchWriterConfig (org.apache.accumulo.core.client.BatchWriterConfig)8
Range (org.apache.accumulo.core.data.Range)8
Map (java.util.Map)7
AccumuloException (org.apache.accumulo.core.client.AccumuloException)7
TableNotFoundException (org.apache.accumulo.core.client.TableNotFoundException)7
Mutation (org.apache.accumulo.core.data.Mutation)7
ColumnVisibility (org.apache.accumulo.core.security.ColumnVisibility)7
AccumuloConnectionParameters (org.apache.hadoop.hive.accumulo.AccumuloConnectionParameters)7
BatchWriter (org.apache.accumulo.core.client.BatchWriter)6
IteratorSettingBuilder (uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder)6
Edge (uk.gov.gchq.gaffer.data.element.Edge)6
