use of org.apache.accumulo.core.client.AccumuloException in project Gaffer by gchq.
the class CoreKeyGroupByAggregatorIteratorTest method testAggregatingMultiplePropertySets.
private void testAggregatingMultiplePropertySets(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
String visibilityString = "public";
try {
// Create edge
final Edge edge = new Edge(TestGroups.EDGE);
edge.setSource("1");
edge.setDestination("2");
edge.setDirected(true);
edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge.putProperty(AccumuloPropertyNames.COUNT, 1);
//THIS EDGE WILL BE REDUCED MEANING ITS CQ (columnQualifier) will only occur once because its key is equal.
final Edge edge2 = new Edge(TestGroups.EDGE);
edge2.setSource("1");
edge2.setDestination("2");
edge2.setDirected(true);
edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge2.putProperty(AccumuloPropertyNames.COUNT, 2);
final Edge edge3 = new Edge(TestGroups.EDGE);
edge3.setSource("1");
edge3.setDestination("2");
edge3.setDirected(true);
edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge3.putProperty(AccumuloPropertyNames.COUNT, 10);
// Accumulo key
final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
// Accumulo values
final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge.getProperties());
final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge2.getProperties());
final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge3.getProperties());
// Create mutation
final Mutation m1 = new Mutation(key.getRow());
m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
final Mutation m2 = new Mutation(key.getRow());
m2.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value2);
final Mutation m3 = new Mutation(key.getRow());
m3.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value3);
final Mutation m4 = new Mutation(key2.getRow());
m4.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value1);
final Mutation m5 = new Mutation(key.getRow());
m5.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value1);
// Write mutation
final BatchWriterConfig writerConfig = new BatchWriterConfig();
writerConfig.setMaxMemory(1000000L);
writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
writerConfig.setMaxWriteThreads(1);
final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
writer.addMutation(m1);
writer.addMutation(m2);
writer.addMutation(m3);
writer.addMutation(m4);
writer.addMutation(m5);
writer.close();
// Read data back and check we get one merged element
final Authorizations authorizations = new Authorizations(visibilityString);
final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().schema(store.getSchema()).view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
scanner.addScanIterator(iteratorSetting);
final Iterator<Entry<Key, Value>> it = scanner.iterator();
final Entry<Key, Value> entry = it.next();
final Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
final Edge expectedEdge = new Edge(TestGroups.EDGE);
expectedEdge.setSource("1");
expectedEdge.setDestination("2");
expectedEdge.setDirected(true);
expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 9);
expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 15);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_1, 0);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_2, 0);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_3, 0);
expectedEdge.putProperty(AccumuloPropertyNames.PROP_4, 0);
assertEquals(expectedEdge, readEdge);
assertEquals(9, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(15, readEdge.getProperty(AccumuloPropertyNames.COUNT));
// Check no more entries
if (it.hasNext()) {
fail("Additional row found.");
}
} catch (AccumuloException | TableNotFoundException e) {
fail(this.getClass().getSimpleName() + " failed with exception: " + e);
}
}
use of org.apache.accumulo.core.client.AccumuloException in project Gaffer by gchq.
the class CoreKeyGroupByAggregatorIteratorTest method shouldAggregateEverythingWhenGroupByIsSetToBlank.
public void shouldAggregateEverythingWhenGroupByIsSetToBlank(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
final String visibilityString = "public";
try {
// Create edge
final Edge edge = new Edge(TestGroups.EDGE);
edge.setSource("1");
edge.setDestination("2");
edge.setDirected(true);
edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge.putProperty(AccumuloPropertyNames.COUNT, 1);
final Edge edge2 = new Edge(TestGroups.EDGE);
edge2.setSource("1");
edge2.setDestination("2");
edge2.setDirected(true);
edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
edge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge2.putProperty(AccumuloPropertyNames.COUNT, 1);
final Edge edge3 = new Edge(TestGroups.EDGE);
edge3.setSource("1");
edge3.setDestination("2");
edge3.setDirected(true);
edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge3.putProperty(AccumuloPropertyNames.COUNT, 1);
//THIS EDGE WILL BE REDUCED MEANING ITS CQ (columnQualifier) will only occur once because its key is equal.
final Edge edge4 = new Edge(TestGroups.EDGE);
edge4.setSource("1");
edge4.setDestination("2");
edge4.setDirected(true);
edge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 4);
edge4.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge4.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge4.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge4.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge4.putProperty(AccumuloPropertyNames.COUNT, 2);
final Edge edge5 = new Edge(TestGroups.EDGE);
edge5.setSource("1");
edge5.setDestination("2");
edge5.setDirected(true);
edge5.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 3);
edge5.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 5);
edge5.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge5.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge5.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge5.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge5.putProperty(AccumuloPropertyNames.COUNT, 10);
final Edge edge6 = new Edge(TestGroups.EDGE);
edge6.setSource("1");
edge6.setDestination("2");
edge6.setDirected(true);
edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 3);
edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 5);
edge6.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_4, 0);
edge6.putProperty(AccumuloPropertyNames.COUNT, 5);
// Accumulo key
final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
final Key key4 = elementConverter.getKeysFromEdge(edge4).getFirst();
final Key key5 = elementConverter.getKeysFromEdge(edge5).getFirst();
final Key key6 = elementConverter.getKeysFromEdge(edge6).getFirst();
// Accumulo values
final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge.getProperties());
final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge2.getProperties());
final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge3.getProperties());
final Value value4 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge4.getProperties());
final Value value5 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge5.getProperties());
final Value value6 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge6.getProperties());
// Create mutation
final Mutation m1 = new Mutation(key.getRow());
m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
final Mutation m2 = new Mutation(key2.getRow());
m2.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value2);
final Mutation m3 = new Mutation(key.getRow());
m3.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value3);
final Mutation m4 = new Mutation(key.getRow());
m4.put(key4.getColumnFamily(), key4.getColumnQualifier(), new ColumnVisibility(key4.getColumnVisibility()), key4.getTimestamp(), value4);
final Mutation m5 = new Mutation(key.getRow());
m5.put(key5.getColumnFamily(), key5.getColumnQualifier(), new ColumnVisibility(key5.getColumnVisibility()), key5.getTimestamp(), value5);
final Mutation m6 = new Mutation(key.getRow());
m6.put(key6.getColumnFamily(), key6.getColumnQualifier(), new ColumnVisibility(key6.getColumnVisibility()), key6.getTimestamp(), value6);
// Write mutation
final BatchWriterConfig writerConfig = new BatchWriterConfig();
writerConfig.setMaxMemory(1000000L);
writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
writerConfig.setMaxWriteThreads(1);
final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
writer.addMutation(m1);
writer.addMutation(m2);
writer.addMutation(m3);
writer.addMutation(m4);
writer.addMutation(m5);
writer.addMutation(m6);
writer.close();
Edge expectedEdge1 = new Edge(TestGroups.EDGE);
expectedEdge1.setSource("1");
expectedEdge1.setDestination("2");
expectedEdge1.setDirected(true);
expectedEdge1.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 5);
expectedEdge1.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 10);
expectedEdge1.putProperty(AccumuloPropertyNames.COUNT, 20);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_1, 0);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_2, 0);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_3, 0);
expectedEdge1.putProperty(AccumuloPropertyNames.PROP_4, 0);
// Read data back and check we get one merged element
final Authorizations authorizations = new Authorizations(visibilityString);
final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
scanner.addScanIterator(iteratorSetting);
final Iterator<Entry<Key, Value>> it = scanner.iterator();
Entry<Key, Value> entry = it.next();
Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
assertEquals(expectedEdge1, readEdge);
assertEquals(5, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(10, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
assertEquals(20, readEdge.getProperty(AccumuloPropertyNames.COUNT));
if (it.hasNext()) {
fail("Additional row found.");
}
} catch (AccumuloException | TableNotFoundException e) {
fail(this.getClass().getSimpleName() + " failed with exception: " + e);
}
}
use of org.apache.accumulo.core.client.AccumuloException in project Gaffer by gchq.
the class IngestUtils method createSplitsFile.
/**
* Get the existing splits from a table in Accumulo and write a splits file.
* The number of splits is returned.
*
* @param conn - An existing connection to an Accumulo instance
* @param table - The table name
* @param fs - The FileSystem in which to create the splits file
* @param splitsFile - A Path for the output splits file
* @param maxSplits - The maximum number of splits
* @return The number of splits in the table
* @throws IOException for any IO issues reading from the file system. Other accumulo exceptions are caught and wrapped in an IOException.
*/
public static int createSplitsFile(final Connector conn, final String table, final FileSystem fs, final Path splitsFile, final int maxSplits) throws IOException {
LOGGER.info("Creating splits file in location {} from table {} with maximum splits {}", splitsFile, table, maxSplits);
// Get the splits from the table
Collection<Text> splits;
try {
splits = conn.tableOperations().listSplits(table, maxSplits);
} catch (TableNotFoundException | AccumuloSecurityException | AccumuloException e) {
throw new IOException(e.getMessage(), e);
}
// This should have returned at most maxSplits splits, but this is not implemented properly in MockInstance.
if (splits.size() > maxSplits) {
if (conn instanceof MockConnector) {
LOGGER.info("Manually reducing the number of splits to {} due to MockInstance not implementing" + " listSplits(table, maxSplits) properly", maxSplits);
} else {
LOGGER.info("Manually reducing the number of splits to {} (number of splits was {})", maxSplits, splits.size());
}
final Collection<Text> filteredSplits = new TreeSet<>();
final int outputEveryNth = splits.size() / maxSplits;
LOGGER.info("Outputting every {}-th split from {} total", outputEveryNth, splits.size());
int i = 0;
for (final Text text : splits) {
if (i % outputEveryNth == 0) {
filteredSplits.add(text);
}
i++;
if (filteredSplits.size() >= maxSplits) {
break;
}
}
splits = filteredSplits;
}
LOGGER.info("Found {} splits from table {}", splits.size(), table);
try (final PrintStream out = new PrintStream(new BufferedOutputStream(fs.create(splitsFile, true)), false, CommonConstants.UTF_8)) {
// Write the splits to file
if (splits.isEmpty()) {
out.close();
return 0;
}
for (final Text split : splits) {
out.println(new String(Base64.encodeBase64(split.getBytes()), CommonConstants.UTF_8));
}
}
return splits.size();
}
use of org.apache.accumulo.core.client.AccumuloException in project Gaffer by gchq.
the class TableUtils method setLocalityGroups.
public static void setLocalityGroups(final AccumuloStore store) throws StoreException {
final String tableName = store.getProperties().getTable();
Map<String, Set<Text>> localityGroups = new HashMap<>();
for (final String group : store.getSchema().getGroups()) {
HashSet<Text> localityGroup = new HashSet<>();
localityGroup.add(new Text(group));
localityGroups.put(group, localityGroup);
}
LOGGER.info("Setting locality groups on table {}", tableName);
try {
store.getConnection().tableOperations().setLocalityGroups(tableName, localityGroups);
} catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
throw new StoreException(e.getMessage(), e);
}
}
use of org.apache.accumulo.core.client.AccumuloException in project hive by apache.
the class AccumuloStorageHandler method preCreateTable.
@Override
public void preCreateTable(Table table) throws MetaException {
boolean isExternal = isExternalTable(table);
if (table.getSd().getLocation() != null) {
throw new MetaException("Location can't be specified for Accumulo");
}
Map<String, String> serdeParams = table.getSd().getSerdeInfo().getParameters();
String columnMapping = serdeParams.get(AccumuloSerDeParameters.COLUMN_MAPPINGS);
if (columnMapping == null) {
throw new MetaException(AccumuloSerDeParameters.COLUMN_MAPPINGS + " missing from SERDEPROPERTIES");
}
try {
String tblName = getTableName(table);
Connector connector = connectionParams.getConnector();
TableOperations tableOpts = connector.tableOperations();
// Attempt to create the table, taking EXTERNAL into consideration
if (!tableOpts.exists(tblName)) {
if (!isExternal) {
tableOpts.create(tblName);
} else {
throw new MetaException("Accumulo table " + tblName + " doesn't exist even though declared external");
}
} else {
if (!isExternal) {
throw new MetaException("Table " + tblName + " already exists in Accumulo. Use CREATE EXTERNAL TABLE to register with Hive.");
}
}
} catch (AccumuloSecurityException e) {
throw new MetaException(StringUtils.stringifyException(e));
} catch (TableExistsException e) {
throw new MetaException(StringUtils.stringifyException(e));
} catch (AccumuloException e) {
throw new MetaException(StringUtils.stringifyException(e));
}
}
Aggregations