use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.
the class AccumuloRetriever method getScanner.
/**
* Create a scanner to use used in your query.
* <p>
*
* @param ranges the ranges to get the scanner for
* @return A {@link org.apache.accumulo.core.client.BatchScanner} for the
* table specified in the properties with the ranges provided.
* @throws TableNotFoundException if an accumulo table could not be found
* @throws StoreException if a connection to accumulo could not be created.
*/
protected BatchScanner getScanner(final Set<Range> ranges) throws TableNotFoundException, StoreException {
final BatchScanner scanner = store.getConnection().createBatchScanner(store.getProperties().getTable(), authorisations, store.getProperties().getThreadsForBatchScanner());
if (iteratorSettings != null) {
for (final IteratorSetting iteratorSetting : iteratorSettings) {
if (iteratorSetting != null) {
scanner.addScanIterator(iteratorSetting);
}
}
}
scanner.setRanges(ranges);
// Currently hard links element class to column family position.
if (IncludeEdgeType.NONE != operation.getIncludeEdges()) {
for (final String col : operation.getView().getEdgeGroups()) {
scanner.fetchColumnFamily(new Text(col));
}
}
if (operation.isIncludeEntities()) {
for (final String col : operation.getView().getEntityGroups()) {
scanner.fetchColumnFamily(new Text(col));
}
}
return scanner;
}
use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.
the class TableUtilsTest method shouldCreateTableWithAllRequiredIterators.
@Test
public void shouldCreateTableWithAllRequiredIterators() throws Exception {
// Given
final MockAccumuloStore store = new MockAccumuloStore();
final Schema schema = new Schema.Builder().type(TestTypes.ID_STRING, new TypeDefinition.Builder().aggregateFunction(new StringConcat()).clazz(String.class).build()).type(TestTypes.DIRECTED_TRUE, Boolean.class).edge(TestGroups.EDGE, new SchemaEdgeDefinition.Builder().source(TestTypes.ID_STRING).destination(TestTypes.ID_STRING).directed(TestTypes.DIRECTED_TRUE).build()).build();
final AccumuloProperties props = AccumuloProperties.loadStoreProperties(StreamUtil.storeProps(TableUtilsTest.class));
props.setTable(TABLE_NAME);
store.initialise(schema, props);
// When
TableUtils.createTable(store);
// Then
final Map<String, EnumSet<IteratorScope>> itrs = store.getConnection().tableOperations().listIterators(TABLE_NAME);
assertEquals(2, itrs.size());
final EnumSet<IteratorScope> validator = itrs.get(AccumuloStoreConstants.VALIDATOR_ITERATOR_NAME);
assertEquals(EnumSet.allOf(IteratorScope.class), validator);
final IteratorSetting validatorSetting = store.getConnection().tableOperations().getIteratorSetting(TABLE_NAME, AccumuloStoreConstants.VALIDATOR_ITERATOR_NAME, IteratorScope.majc);
assertEquals(AccumuloStoreConstants.VALIDATOR_ITERATOR_PRIORITY, validatorSetting.getPriority());
assertEquals(ValidatorFilter.class.getName(), validatorSetting.getIteratorClass());
final Map<String, String> validatorOptions = validatorSetting.getOptions();
assertNotNull(Schema.fromJson(validatorOptions.get(AccumuloStoreConstants.SCHEMA).getBytes(CommonConstants.UTF_8)).getEdge(TestGroups.EDGE));
assertEquals(ByteEntityAccumuloElementConverter.class.getName(), validatorOptions.get(AccumuloStoreConstants.ACCUMULO_ELEMENT_CONVERTER_CLASS));
final EnumSet<IteratorScope> aggregator = itrs.get(AccumuloStoreConstants.AGGREGATOR_ITERATOR_NAME);
assertEquals(EnumSet.allOf(IteratorScope.class), aggregator);
final IteratorSetting aggregatorSetting = store.getConnection().tableOperations().getIteratorSetting(TABLE_NAME, AccumuloStoreConstants.AGGREGATOR_ITERATOR_NAME, IteratorScope.majc);
assertEquals(AccumuloStoreConstants.AGGREGATOR_ITERATOR_PRIORITY, aggregatorSetting.getPriority());
assertEquals(AggregatorIterator.class.getName(), aggregatorSetting.getIteratorClass());
final Map<String, String> aggregatorOptions = aggregatorSetting.getOptions();
assertNotNull(Schema.fromJson(aggregatorOptions.get(AccumuloStoreConstants.SCHEMA).getBytes(CommonConstants.UTF_8)).getEdge(TestGroups.EDGE));
assertEquals(ByteEntityAccumuloElementConverter.class.getName(), aggregatorOptions.get(AccumuloStoreConstants.ACCUMULO_ELEMENT_CONVERTER_CLASS));
final Map<String, String> tableProps = new HashMap<>();
for (final Map.Entry<String, String> entry : store.getConnection().tableOperations().getProperties(TABLE_NAME)) {
tableProps.put(entry.getKey(), entry.getValue());
}
assertEquals(0, Integer.parseInt(tableProps.get(Property.TABLE_FILE_REPLICATION.getKey())));
}
use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.
the class TableUtilsTest method shouldCreateTableCorrectlyIfSchemaContainsNoAggregators.
@Test
public void shouldCreateTableCorrectlyIfSchemaContainsNoAggregators() throws Exception {
// Given
final MockAccumuloStore store = new MockAccumuloStore();
final Schema schema = new Schema.Builder().type(TestTypes.ID_STRING, String.class).type(TestTypes.DIRECTED_TRUE, Boolean.class).edge(TestGroups.EDGE, new SchemaEdgeDefinition.Builder().source(TestTypes.ID_STRING).destination(TestTypes.ID_STRING).directed(TestTypes.DIRECTED_TRUE).build()).build();
final AccumuloProperties props = AccumuloProperties.loadStoreProperties(StreamUtil.storeProps(TableUtilsTest.class));
props.setTable(NO_AGGREGATORS_TABLE_NAME);
store.initialise(schema, props);
// When
TableUtils.createTable(store);
// Then
final Map<String, EnumSet<IteratorScope>> itrs = store.getConnection().tableOperations().listIterators(NO_AGGREGATORS_TABLE_NAME);
assertEquals(1, itrs.size());
final EnumSet<IteratorScope> validator = itrs.get(AccumuloStoreConstants.VALIDATOR_ITERATOR_NAME);
assertEquals(EnumSet.allOf(IteratorScope.class), validator);
final IteratorSetting validatorSetting = store.getConnection().tableOperations().getIteratorSetting(NO_AGGREGATORS_TABLE_NAME, AccumuloStoreConstants.VALIDATOR_ITERATOR_NAME, IteratorScope.majc);
assertEquals(AccumuloStoreConstants.VALIDATOR_ITERATOR_PRIORITY, validatorSetting.getPriority());
assertEquals(ValidatorFilter.class.getName(), validatorSetting.getIteratorClass());
final Map<String, String> validatorOptions = validatorSetting.getOptions();
assertNotNull(Schema.fromJson(validatorOptions.get(AccumuloStoreConstants.SCHEMA).getBytes(CommonConstants.UTF_8)).getEdge(TestGroups.EDGE));
assertEquals(ByteEntityAccumuloElementConverter.class.getName(), validatorOptions.get(AccumuloStoreConstants.ACCUMULO_ELEMENT_CONVERTER_CLASS));
final EnumSet<IteratorScope> aggregator = itrs.get(AccumuloStoreConstants.AGGREGATOR_ITERATOR_NAME);
assertNull(aggregator);
final IteratorSetting aggregatorSetting = store.getConnection().tableOperations().getIteratorSetting(NO_AGGREGATORS_TABLE_NAME, AccumuloStoreConstants.AGGREGATOR_ITERATOR_NAME, IteratorScope.majc);
assertNull(aggregatorSetting);
final Map<String, String> tableProps = new HashMap<>();
for (final Map.Entry<String, String> entry : store.getConnection().tableOperations().getProperties(NO_AGGREGATORS_TABLE_NAME)) {
tableProps.put(entry.getKey(), entry.getValue());
}
assertEquals(0, Integer.parseInt(tableProps.get(Property.TABLE_FILE_REPLICATION.getKey())));
}
use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.
the class CoreKeyGroupByAggregatorIteratorTest method testAggregatingEmptyColumnQualifier.
public void testAggregatingEmptyColumnQualifier(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
final String visibilityString = "public";
try {
// Create edge
final Edge edge = new Edge(TestGroups.EDGE);
edge.setSource("1");
edge.setDestination("2");
edge.setDirected(true);
edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge.putProperty(AccumuloPropertyNames.COUNT, 1);
//THIS EDGE WILL BE REDUCED MEANING ITS CQ (columnQualifier) will only occur once because its key is equal.
final Edge edge2 = new Edge(TestGroups.EDGE);
edge2.setSource("1");
edge2.setDestination("2");
edge2.setDirected(true);
edge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge2.putProperty(AccumuloPropertyNames.COUNT, 2);
final Edge edge3 = new Edge(TestGroups.EDGE);
edge3.setSource("1");
edge3.setDestination("2");
edge3.setDirected(true);
edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge3.putProperty(AccumuloPropertyNames.COUNT, 10);
// Accumulo key
final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
// Accumulo values
final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge.getProperties());
final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge2.getProperties());
final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge3.getProperties());
// Create mutation
final Mutation m1 = new Mutation(key.getRow());
m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
final Mutation m2 = new Mutation(key.getRow());
m2.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value2);
final Mutation m3 = new Mutation(key.getRow());
m3.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value3);
final Mutation m4 = new Mutation(key2.getRow());
m4.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value1);
final Mutation m5 = new Mutation(key.getRow());
m5.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value1);
// Write mutation
final BatchWriterConfig writerConfig = new BatchWriterConfig();
writerConfig.setMaxMemory(1000000L);
writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
writerConfig.setMaxWriteThreads(1);
final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
writer.addMutation(m1);
writer.addMutation(m2);
writer.addMutation(m3);
writer.addMutation(m4);
writer.addMutation(m5);
writer.close();
Edge expectedEdge = new Edge(TestGroups.EDGE);
expectedEdge.setSource("1");
expectedEdge.setDestination("2");
expectedEdge.setDirected(true);
expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 15);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_1, 0);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_2, 0);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_3, 0);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_4, 0);
// Read data back and check we get one merged element
final Authorizations authorizations = new Authorizations(visibilityString);
final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
scanner.addScanIterator(iteratorSetting);
final Iterator<Entry<Key, Value>> it = scanner.iterator();
final Entry<Key, Value> entry = it.next();
final Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
assertEquals(expectedEdge, readEdge);
assertEquals(8, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(15, readEdge.getProperty(AccumuloPropertyNames.COUNT));
// Check no more entries
if (it.hasNext()) {
fail("Additional row found.");
}
} catch (AccumuloException | TableNotFoundException e) {
fail(this.getClass().getSimpleName() + " failed with exception: " + e);
}
}
use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.
the class CoreKeyGroupByAggregatorIteratorTest method testAggregatingSinglePropertySet.
public void testAggregatingSinglePropertySet(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
String visibilityString = "public";
try {
// Create edge
final Edge edge = new Edge(TestGroups.EDGE);
edge.setSource("1");
edge.setDestination("2");
edge.setDirected(true);
edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
edge.putProperty(AccumuloPropertyNames.COUNT, 1);
final Properties properties1 = new Properties();
properties1.put(AccumuloPropertyNames.COUNT, 1);
// Accumulo key
final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
// Accumulo values
final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties1);
// Create mutation
final Mutation m1 = new Mutation(key.getRow());
m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
// Write mutation
final BatchWriterConfig writerConfig = new BatchWriterConfig();
writerConfig.setMaxMemory(1000000L);
writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
writerConfig.setMaxWriteThreads(1);
final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
writer.addMutation(m1);
writer.close();
final Edge expectedEdge = new Edge(TestGroups.EDGE);
expectedEdge.setSource("1");
expectedEdge.setDestination("2");
expectedEdge.setDirected(true);
expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 1);
// Read data back and check we get one merged element
final Authorizations authorizations = new Authorizations(visibilityString);
final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
scanner.addScanIterator(iteratorSetting);
final Iterator<Entry<Key, Value>> it = scanner.iterator();
final Entry<Key, Value> entry = it.next();
final Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
assertEquals(expectedEdge, readEdge);
assertEquals(8, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(1, readEdge.getProperty(AccumuloPropertyNames.COUNT));
// Check no more entries
if (it.hasNext()) {
fail("Additional row found.");
}
} catch (AccumuloException | TableNotFoundException e) {
fail(this.getClass().getSimpleName() + " failed with exception: " + e);
}
}
Aggregations