Examples with AccumuloException org.apache.accumulo.core.client.AccumuloException used on opensource projects

Example 11 with AccumuloException

use of org.apache.accumulo.core.client.AccumuloException in project Gaffer by gchq.

the class AccumuloKeyRangePartitioner method getSplits.

public static synchronized String[] getSplits(final AccumuloStore store) throws OperationException {
    final Connector connector;
    try {
        connector = store.getConnection();
    } catch (StoreException e) {
        throw new OperationException("Failed to create accumulo connection", e);
    }
    final String table = store.getProperties().getTable();
    try {
        final Collection<Text> splits = connector.tableOperations().listSplits(table);
        final String[] arr = new String[splits.size()];
        return splits.parallelStream().map(text -> text.toString()).collect(Collectors.toList()).toArray(arr);
    } catch (TableNotFoundException | AccumuloSecurityException | AccumuloException e) {
        throw new OperationException("Failed to get accumulo split points from table " + table, e);
    }
}

Example 12 with AccumuloException

use of org.apache.accumulo.core.client.AccumuloException in project Gaffer by gchq.

the class RowIdAggregatorTest method testAggregatingMultiplePropertySetsAcrossRowIDRange.

private void testAggregatingMultiplePropertySetsAcrossRowIDRange(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException, RangeFactoryException {
    String visibilityString = "public";
    try {
        // Create table
        // (this method creates the table, removes the versioning iterator, and adds the SetOfStatisticsCombiner iterator).
        TableUtils.createTable(store);
        final Properties properties1 = new Properties();
        properties1.put(AccumuloPropertyNames.COUNT, 1);
        final Properties properties2 = new Properties();
        properties2.put(AccumuloPropertyNames.COUNT, 1);
        final Properties properties3 = new Properties();
        properties3.put(AccumuloPropertyNames.COUNT, 2);
        // Create edge
        final Edge edge = new Edge(TestGroups.EDGE);
        edge.setSource("2");
        edge.setDestination("1");
        edge.setDirected(true);
        edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
        final Edge edge2 = new Edge(TestGroups.EDGE);
        edge2.setSource("B");
        edge2.setDestination("Z");
        edge2.setDirected(true);
        edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge2.putProperty(AccumuloPropertyNames.PROP_1, 1);
        edge2.putProperty(AccumuloPropertyNames.PROP_2, 1);
        edge2.putProperty(AccumuloPropertyNames.PROP_3, 1);
        edge2.putProperty(AccumuloPropertyNames.PROP_4, 1);
        final Edge edge3 = new Edge(TestGroups.EDGE);
        edge3.setSource("3");
        edge3.setDestination("8");
        edge3.setDirected(true);
        edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
        final Edge edge6 = new Edge("BasicEdge2");
        edge6.setSource("1");
        edge6.setDestination("5");
        edge6.setDirected(true);
        edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
        edge6.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge6.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge6.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge6.putProperty(AccumuloPropertyNames.PROP_4, 0);
        final Edge edge7 = new Edge("BasicEdge2");
        edge7.setSource("2");
        edge7.setDestination("6");
        edge7.setDirected(true);
        edge7.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge7.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge7.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge7.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge7.putProperty(AccumuloPropertyNames.PROP_4, 0);
        final Edge edge8 = new Edge("BasicEdge2");
        edge8.setSource("4");
        edge8.setDestination("8");
        edge8.setDirected(true);
        edge8.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
        edge8.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge8.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge8.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge8.putProperty(AccumuloPropertyNames.PROP_4, 0);
        final Edge edge9 = new Edge("BasicEdge2");
        edge9.setSource("5");
        edge9.setDestination("9");
        edge9.setDirected(true);
        edge9.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
        edge9.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge9.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge9.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge9.putProperty(AccumuloPropertyNames.PROP_4, 0);
        // Accumulo key
        final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
        final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
        final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
        final Key key4 = elementConverter.getKeysFromEdge(edge6).getFirst();
        final Key key5 = elementConverter.getKeysFromEdge(edge7).getFirst();
        final Key key6 = elementConverter.getKeysFromEdge(edge8).getFirst();
        final Key key7 = elementConverter.getKeysFromEdge(edge9).getFirst();
        // Accumulo values
        final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties1);
        final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties2);
        final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties3);
        final Value value4 = elementConverter.getValueFromProperties(TestGroups.EDGE_2, properties1);
        final Value value5 = elementConverter.getValueFromProperties(TestGroups.EDGE_2, properties2);
        //Create mutation
        final Mutation m1 = new Mutation(key.getRow());
        m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
        final Mutation m2 = new Mutation(key.getRow());
        m2.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value2);
        final Mutation m3 = new Mutation(key.getRow());
        m3.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value3);
        final Mutation m4 = new Mutation(key2.getRow());
        m4.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value1);
        final Mutation m5 = new Mutation(key.getRow());
        m5.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value1);
        final Mutation m6 = new Mutation(key4.getRow());
        m6.put(key4.getColumnFamily(), key4.getColumnQualifier(), new ColumnVisibility(key4.getColumnVisibility()), key4.getTimestamp(), value4);
        final Mutation m7 = new Mutation(key5.getRow());
        m7.put(key5.getColumnFamily(), key5.getColumnQualifier(), new ColumnVisibility(key5.getColumnVisibility()), key5.getTimestamp(), value5);
        final Mutation m8 = new Mutation(key6.getRow());
        m8.put(key6.getColumnFamily(), key6.getColumnQualifier(), new ColumnVisibility(key6.getColumnVisibility()), key6.getTimestamp(), value5);
        final Mutation m9 = new Mutation(key7.getRow());
        m9.put(key7.getColumnFamily(), key7.getColumnQualifier(), new ColumnVisibility(key7.getColumnVisibility()), key7.getTimestamp(), value5);
        // Write mutation
        final BatchWriterConfig writerConfig = new BatchWriterConfig();
        writerConfig.setMaxMemory(1000000L);
        writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
        writerConfig.setMaxWriteThreads(1);
        final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
        writer.addMutation(m1);
        writer.addMutation(m2);
        writer.addMutation(m3);
        writer.addMutation(m4);
        writer.addMutation(m5);
        writer.addMutation(m6);
        writer.addMutation(m7);
        writer.addMutation(m8);
        writer.addMutation(m9);
        writer.close();
        // Read data back and check we get one merged element
        final Authorizations authorizations = new Authorizations(visibilityString);
        final BatchScanner scanner = store.getConnection().createBatchScanner(store.getProperties().getTable(), authorizations, 1000);
        try {
            scanner.addScanIterator(store.getKeyPackage().getIteratorFactory().getRowIDAggregatorIteratorSetting(store, "BasicEdge2"));
        } catch (IteratorSettingException e) {
            fail(e.getMessage());
        }
        final RangeFactory rangeF = store.getKeyPackage().getRangeFactory();
        final Range r = rangeF.getRangeFromPair(new Pair<ElementSeed>((new EntitySeed("1")), new EntitySeed("4")), new SummariseGroupOverRanges());
        final Range r2 = rangeF.getRangeFromPair(new Pair<ElementSeed>((new EntitySeed("5")), new EntitySeed("5")), new SummariseGroupOverRanges());
        scanner.setRanges(Arrays.asList(r, r2));
        final Iterator<Entry<Key, Value>> it = scanner.iterator();
        Entry<Key, Value> entry = it.next();
        Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        Edge expectedEdge = new Edge("BasicEdge2");
        expectedEdge.setSource("4");
        expectedEdge.setDestination("8");
        expectedEdge.setDirected(true);
        expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 5);
        expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 3);
        assertEquals(expectedEdge, readEdge);
        assertEquals(5, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(3, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        // Check we get the Result of the second provided range
        assertTrue(it.hasNext());
        entry = it.next();
        readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        expectedEdge = new Edge("BasicEdge2");
        expectedEdge.setSource("5");
        expectedEdge.setDestination("9");
        expectedEdge.setDirected(true);
        expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
        expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 1);
        assertEquals(expectedEdge, readEdge);
        //Check no additional rows are found. (For a table of this size we shouldn't see this)
        if (it.hasNext()) {
            fail("Additional row found.");
        }
    } catch (AccumuloException | TableExistsException | TableNotFoundException e) {
        fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    }
}

Example 13 with AccumuloException

use of org.apache.accumulo.core.client.AccumuloException in project Gaffer by gchq.

the class CoreKeyGroupByAggregatorIteratorTest method testAggregatingEmptyColumnQualifier.

public void testAggregatingEmptyColumnQualifier(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
    final String visibilityString = "public";
    try {
        // Create edge
        final Edge edge = new Edge(TestGroups.EDGE);
        edge.setSource("1");
        edge.setDestination("2");
        edge.setDirected(true);
        edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
        edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge.putProperty(AccumuloPropertyNames.COUNT, 1);
        //THIS EDGE WILL BE REDUCED MEANING ITS CQ (columnQualifier) will only occur once because its key is equal.
        final Edge edge2 = new Edge(TestGroups.EDGE);
        edge2.setSource("1");
        edge2.setDestination("2");
        edge2.setDirected(true);
        edge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge2.putProperty(AccumuloPropertyNames.COUNT, 2);
        final Edge edge3 = new Edge(TestGroups.EDGE);
        edge3.setSource("1");
        edge3.setDestination("2");
        edge3.setDirected(true);
        edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge3.putProperty(AccumuloPropertyNames.COUNT, 10);
        // Accumulo key
        final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
        final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
        final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
        // Accumulo values
        final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge.getProperties());
        final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge2.getProperties());
        final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge3.getProperties());
        // Create mutation
        final Mutation m1 = new Mutation(key.getRow());
        m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
        final Mutation m2 = new Mutation(key.getRow());
        m2.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value2);
        final Mutation m3 = new Mutation(key.getRow());
        m3.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value3);
        final Mutation m4 = new Mutation(key2.getRow());
        m4.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value1);
        final Mutation m5 = new Mutation(key.getRow());
        m5.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value1);
        // Write mutation
        final BatchWriterConfig writerConfig = new BatchWriterConfig();
        writerConfig.setMaxMemory(1000000L);
        writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
        writerConfig.setMaxWriteThreads(1);
        final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
        writer.addMutation(m1);
        writer.addMutation(m2);
        writer.addMutation(m3);
        writer.addMutation(m4);
        writer.addMutation(m5);
        writer.close();
        Edge expectedEdge = new Edge(TestGroups.EDGE);
        expectedEdge.setSource("1");
        expectedEdge.setDestination("2");
        expectedEdge.setDirected(true);
        expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
        expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 15);
        expectedEdge.putProperty(AccumuloPropertyNames.PROP_1, 0);
        expectedEdge.putProperty(AccumuloPropertyNames.PROP_2, 0);
        expectedEdge.putProperty(AccumuloPropertyNames.PROP_3, 0);
        expectedEdge.putProperty(AccumuloPropertyNames.PROP_4, 0);
        // Read data back and check we get one merged element
        final Authorizations authorizations = new Authorizations(visibilityString);
        final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
        final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
        scanner.addScanIterator(iteratorSetting);
        final Iterator<Entry<Key, Value>> it = scanner.iterator();
        final Entry<Key, Value> entry = it.next();
        final Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        assertEquals(expectedEdge, readEdge);
        assertEquals(8, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(15, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        // Check no more entries
        if (it.hasNext()) {
            fail("Additional row found.");
        }
    } catch (AccumuloException | TableNotFoundException e) {
        fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    }
}

Example 14 with AccumuloException

use of org.apache.accumulo.core.client.AccumuloException in project Gaffer by gchq.

the class CoreKeyGroupByAggregatorIteratorTest method testAggregatingSinglePropertySet.

public void testAggregatingSinglePropertySet(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
    String visibilityString = "public";
    try {
        // Create edge
        final Edge edge = new Edge(TestGroups.EDGE);
        edge.setSource("1");
        edge.setDestination("2");
        edge.setDirected(true);
        edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
        edge.putProperty(AccumuloPropertyNames.COUNT, 1);
        final Properties properties1 = new Properties();
        properties1.put(AccumuloPropertyNames.COUNT, 1);
        // Accumulo key
        final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
        // Accumulo values
        final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties1);
        // Create mutation
        final Mutation m1 = new Mutation(key.getRow());
        m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
        // Write mutation
        final BatchWriterConfig writerConfig = new BatchWriterConfig();
        writerConfig.setMaxMemory(1000000L);
        writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
        writerConfig.setMaxWriteThreads(1);
        final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
        writer.addMutation(m1);
        writer.close();
        final Edge expectedEdge = new Edge(TestGroups.EDGE);
        expectedEdge.setSource("1");
        expectedEdge.setDestination("2");
        expectedEdge.setDirected(true);
        expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
        expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 1);
        // Read data back and check we get one merged element
        final Authorizations authorizations = new Authorizations(visibilityString);
        final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
        final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
        scanner.addScanIterator(iteratorSetting);
        final Iterator<Entry<Key, Value>> it = scanner.iterator();
        final Entry<Key, Value> entry = it.next();
        final Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        assertEquals(expectedEdge, readEdge);
        assertEquals(8, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(1, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        // Check no more entries
        if (it.hasNext()) {
            fail("Additional row found.");
        }
    } catch (AccumuloException | TableNotFoundException e) {
        fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    }
}

Example 15 with AccumuloException

use of org.apache.accumulo.core.client.AccumuloException in project Gaffer by gchq.

the class SplitTableTool method run.

@Override
public int run(final String[] arg0) throws OperationException {
    LOGGER.info("Running SplitTableTool");
    final Configuration conf = getConf();
    FileSystem fs;
    try {
        fs = FileSystem.get(conf);
    } catch (final IOException e) {
        throw new OperationException("Failed to get Filesystem from configuration: " + e.getMessage(), e);
    }
    final SortedSet<Text> splits = new TreeSet<>();
    try (final BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(new Path(operation.getInputPath())), CommonConstants.UTF_8))) {
        String line = br.readLine();
        while (line != null) {
            splits.add(new Text(line));
            line = br.readLine();
        }
    } catch (final IOException e) {
        throw new OperationException(e.getMessage(), e);
    }
    try {
        store.getConnection().tableOperations().addSplits(store.getProperties().getTable(), splits);
        LOGGER.info("Added {} splits to table {}", splits.size(), store.getProperties().getTable());
    } catch (final TableNotFoundException | AccumuloException | AccumuloSecurityException | StoreException e) {
        LOGGER.error("Failed to add {} split points to table {}", splits.size(), store.getProperties().getTable());
        throw new OperationException("Failed to add split points to the table specified: " + e.getMessage(), e);
    }
    return SUCCESS_RESPONSE;
}