Examples with IteratorSettingException uk.gov.gchq.gaffer.accumulostore.key.exception.IteratorSettingException used on opensource projects

Example 21 with IteratorSettingException

use of uk.gov.gchq.gaffer.accumulostore.key.exception.IteratorSettingException in project Gaffer by gchq.

the class AccumuloSingleIDRetrieverTest method testUndirectedEdgeIdQueries.

private void testUndirectedEdgeIdQueries(final AccumuloStore store) throws StoreException {
    setupGraph(store, NUM_ENTRIES);
    final User user = new User();
    // Create set to query for
    final Set<ElementId> ids = new HashSet<>();
    for (int i = 0; i < NUM_ENTRIES; i++) {
        ids.add(new EdgeSeed("" + i, "B", false));
        ids.add(new EdgeSeed("" + i, "C", true));
    }
    final View view = new View.Builder().edge(TestGroups.EDGE).build();
    AccumuloSingleIDRetriever<?> retriever = null;
    final GetElements operation = new GetElements.Builder().view(view).input(ids).build();
    operation.setDirectedType(DirectedType.UNDIRECTED);
    try {
        retriever = new AccumuloSingleIDRetriever<>(store, operation, user);
    } catch (final IteratorSettingException e) {
        throw new RuntimeException(e);
    }
    for (final Element element : retriever) {
        Edge edge = (Edge) element;
        assertEquals("B", edge.getDestination());
    }
    // We should have only 1000 returned the i-B edges that are undirected
    assertEquals(NUM_ENTRIES, Iterables.size(retriever));
}

Example 22 with IteratorSettingException

use of uk.gov.gchq.gaffer.accumulostore.key.exception.IteratorSettingException in project Gaffer by gchq.

the class AccumuloSingleIDRetrieverTest method testEntityIdQueryEdgesOnly.

private void testEntityIdQueryEdgesOnly(final AccumuloStore store) throws StoreException {
    setupGraph(store, NUM_ENTRIES);
    final User user = new User();
    // Create set to query for
    final Set<ElementId> ids = new HashSet<>();
    for (int i = 0; i < NUM_ENTRIES; i++) {
        ids.add(new EntitySeed("" + i));
    }
    final View view = new View.Builder().edge(TestGroups.EDGE).build();
    AccumuloSingleIDRetriever retriever = null;
    final GetElements operation = new GetElements.Builder().view(view).input(ids).build();
    try {
        retriever = new AccumuloSingleIDRetriever(store, operation, user);
    } catch (final IteratorSettingException e) {
        throw new RuntimeException(e);
    }
    // Should find both i-B and i-C edges.
    assertEquals(NUM_ENTRIES * 2, Iterables.size(retriever));
}

Example 23 with IteratorSettingException

use of uk.gov.gchq.gaffer.accumulostore.key.exception.IteratorSettingException in project Gaffer by gchq.

the class RowIdAggregatorTest method testAggregatingMultiplePropertySetsAcrossRowIDRange.

private void testAggregatingMultiplePropertySetsAcrossRowIDRange(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, RangeFactoryException {
    String visibilityString = "public";
    BatchScanner scanner = null;
    try {
        // Create table
        // (this method creates the table, removes the versioning iterator, and adds the SetOfStatisticsCombiner iterator).
        TableUtils.createTable(store);
        final Properties properties1 = new Properties();
        properties1.put(AccumuloPropertyNames.COUNT, 1);
        final Properties properties2 = new Properties();
        properties2.put(AccumuloPropertyNames.COUNT, 1);
        final Properties properties3 = new Properties();
        properties3.put(AccumuloPropertyNames.COUNT, 2);
        // Create edge
        final Edge edge = new Edge.Builder().group(TestGroups.EDGE).source("2").dest("1").directed(true).property(AccumuloPropertyNames.COLUMN_QUALIFIER, 1).property(AccumuloPropertyNames.PROP_1, 0).property(AccumuloPropertyNames.PROP_2, 0).property(AccumuloPropertyNames.PROP_3, 0).property(AccumuloPropertyNames.PROP_4, 0).build();
        final Edge edge2 = new Edge.Builder().group(TestGroups.EDGE).source("B").dest("Z").directed(true).property(AccumuloPropertyNames.COLUMN_QUALIFIER, 1).property(AccumuloPropertyNames.PROP_1, 1).property(AccumuloPropertyNames.PROP_2, 1).property(AccumuloPropertyNames.PROP_3, 1).property(AccumuloPropertyNames.PROP_4, 1).build();
        final Edge edge3 = new Edge.Builder().group(TestGroups.EDGE).source("3").dest("8").directed(true).property(AccumuloPropertyNames.COLUMN_QUALIFIER, 1).property(AccumuloPropertyNames.PROP_1, 0).property(AccumuloPropertyNames.PROP_2, 0).property(AccumuloPropertyNames.PROP_3, 0).property(AccumuloPropertyNames.PROP_4, 0).build();
        final Edge edge6 = new Edge.Builder().group("BasicEdge2").source("1").dest("5").directed(true).property(AccumuloPropertyNames.COLUMN_QUALIFIER, 2).property(AccumuloPropertyNames.PROP_1, 0).property(AccumuloPropertyNames.PROP_2, 0).property(AccumuloPropertyNames.PROP_3, 0).property(AccumuloPropertyNames.PROP_4, 0).build();
        final Edge edge7 = new Edge.Builder().group("BasicEdge2").source("2").dest("6").directed(true).property(AccumuloPropertyNames.COLUMN_QUALIFIER, 1).property(AccumuloPropertyNames.PROP_1, 0).property(AccumuloPropertyNames.PROP_2, 0).property(AccumuloPropertyNames.PROP_3, 0).property(AccumuloPropertyNames.PROP_4, 0).build();
        final Edge edge8 = new Edge.Builder().group("BasicEdge2").source("4").dest("8").directed(true).property(AccumuloPropertyNames.COLUMN_QUALIFIER, 2).property(AccumuloPropertyNames.PROP_1, 0).property(AccumuloPropertyNames.PROP_2, 0).property(AccumuloPropertyNames.PROP_3, 0).property(AccumuloPropertyNames.PROP_4, 0).build();
        final Edge edge9 = new Edge.Builder().group("BasicEdge2").source("5").dest("9").directed(true).property(AccumuloPropertyNames.COLUMN_QUALIFIER, 2).property(AccumuloPropertyNames.PROP_1, 0).property(AccumuloPropertyNames.PROP_2, 0).property(AccumuloPropertyNames.PROP_3, 0).property(AccumuloPropertyNames.PROP_4, 0).build();
        // Accumulo key
        final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
        final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
        final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
        final Key key4 = elementConverter.getKeysFromEdge(edge6).getFirst();
        final Key key5 = elementConverter.getKeysFromEdge(edge7).getFirst();
        final Key key6 = elementConverter.getKeysFromEdge(edge8).getFirst();
        final Key key7 = elementConverter.getKeysFromEdge(edge9).getFirst();
        // Accumulo values
        final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties1);
        final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties2);
        final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties3);
        final Value value4 = elementConverter.getValueFromProperties(TestGroups.EDGE_2, properties1);
        final Value value5 = elementConverter.getValueFromProperties(TestGroups.EDGE_2, properties2);
        // Create mutation
        final Mutation m1 = new Mutation(key.getRow());
        m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
        final Mutation m2 = new Mutation(key.getRow());
        m2.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value2);
        final Mutation m3 = new Mutation(key.getRow());
        m3.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value3);
        final Mutation m4 = new Mutation(key2.getRow());
        m4.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value1);
        final Mutation m5 = new Mutation(key.getRow());
        m5.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value1);
        final Mutation m6 = new Mutation(key4.getRow());
        m6.put(key4.getColumnFamily(), key4.getColumnQualifier(), new ColumnVisibility(key4.getColumnVisibility()), key4.getTimestamp(), value4);
        final Mutation m7 = new Mutation(key5.getRow());
        m7.put(key5.getColumnFamily(), key5.getColumnQualifier(), new ColumnVisibility(key5.getColumnVisibility()), key5.getTimestamp(), value5);
        final Mutation m8 = new Mutation(key6.getRow());
        m8.put(key6.getColumnFamily(), key6.getColumnQualifier(), new ColumnVisibility(key6.getColumnVisibility()), key6.getTimestamp(), value5);
        final Mutation m9 = new Mutation(key7.getRow());
        m9.put(key7.getColumnFamily(), key7.getColumnQualifier(), new ColumnVisibility(key7.getColumnVisibility()), key7.getTimestamp(), value5);
        // Write mutation
        final BatchWriterConfig writerConfig = new BatchWriterConfig();
        writerConfig.setMaxMemory(1000000L);
        writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
        writerConfig.setMaxWriteThreads(1);
        final BatchWriter writer = store.getConnection().createBatchWriter(store.getTableName(), writerConfig);
        writer.addMutation(m1);
        writer.addMutation(m2);
        writer.addMutation(m3);
        writer.addMutation(m4);
        writer.addMutation(m5);
        writer.addMutation(m6);
        writer.addMutation(m7);
        writer.addMutation(m8);
        writer.addMutation(m9);
        writer.close();
        // Read data back and check we get one merged element
        final Authorizations authorizations = new Authorizations(visibilityString);
        scanner = store.getConnection().createBatchScanner(store.getTableName(), authorizations, 1000);
        try {
            scanner.addScanIterator(store.getKeyPackage().getIteratorFactory().getRowIDAggregatorIteratorSetting(store, "BasicEdge2"));
        } catch (final IteratorSettingException e) {
            fail(e.getMessage());
        }
        final RangeFactory rangeF = store.getKeyPackage().getRangeFactory();
        final SummariseGroupOverRanges summariseGroupOverRanges = new SummariseGroupOverRanges.Builder().view(new View.Builder().edge("BasicEdge2").entity("BasicEntity").build()).build();
        final Range r = rangeF.getRangeFromPair(new Pair<>(new EntitySeed("1"), new EntitySeed("4")), summariseGroupOverRanges);
        final Range r2 = rangeF.getRangeFromPair(new Pair<>(new EntitySeed("5"), new EntitySeed("5")), summariseGroupOverRanges);
        scanner.setRanges(Arrays.asList(r, r2));
        final Iterator<Entry<Key, Value>> it = scanner.iterator();
        Entry<Key, Value> entry = it.next();
        Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue(), false);
        Edge expectedEdge = new Edge.Builder().group("BasicEdge2").source("4").dest("8").directed(true).property(AccumuloPropertyNames.COLUMN_QUALIFIER, 5).build();
        expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 3);
        assertEquals(expectedEdge, readEdge);
        assertEquals(5, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(3, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        // Check we get the Result of the second provided range
        assertThat(it).hasNext();
        entry = it.next();
        readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue(), false);
        expectedEdge = new Edge.Builder().group("BasicEdge2").source("5").dest("9").directed(true).property(AccumuloPropertyNames.COLUMN_QUALIFIER, 2).property(AccumuloPropertyNames.COUNT, 1).build();
        assertEquals(expectedEdge, readEdge);
        // Check no additional rows are found. (For a table of this size we shouldn't see this)
        if (it.hasNext()) {
            fail("Additional row found.");
        }
    } catch (final AccumuloException | TableExistsException | TableNotFoundException e) {
        fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    } finally {
        if (scanner != null) {
            scanner.close();
        }
    }
}

Example 24 with IteratorSettingException

use of uk.gov.gchq.gaffer.accumulostore.key.exception.IteratorSettingException in project Gaffer by gchq.

the class TableUtils method createTable.

/**
 * Creates a table for Gaffer data and enables the correct Bloom filter;
 * removes the versioning iterator and adds an aggregator Iterator the
 * {@link org.apache.accumulo.core.iterators.user.AgeOffFilter} for the
 * specified time period.
 *
 * @param store the accumulo store
 * @throws StoreException       failure to create accumulo connection or add iterator settings
 * @throws TableExistsException failure to create table
 */
public static synchronized void createTable(final AccumuloStore store) throws StoreException, TableExistsException {
    // Create table and namespace (if provided)
    final String tableName = store.getTableName();
    if (null == tableName) {
        throw new AccumuloRuntimeException("Table name is required.");
    }
    final Connector connector = store.getConnection();
    if (connector.tableOperations().exists(tableName)) {
        LOGGER.info("Table {} exists, not creating", tableName);
        return;
    }
    try {
        final String namespace = store.getProperties().getNamespace();
        if (StringUtils.isNotBlank(namespace)) {
            if (!connector.namespaceOperations().exists(namespace)) {
                LOGGER.info("Creating namespace {} as user {}", namespace, connector.whoami());
                try {
                    connector.namespaceOperations().create(namespace);
                } catch (final NamespaceExistsException e) {
                // This method is synchronised, if you are using the same store only
                // through one client in one JVM you shouldn't get here
                // Someone else got there first, never mind...
                }
            }
        }
        LOGGER.info("Creating table {} as user {}", tableName, connector.whoami());
        connector.tableOperations().create(tableName);
        final String repFactor = store.getProperties().getTableFileReplicationFactor();
        if (null != repFactor) {
            LOGGER.info("Table file replication set to {} on table {}", repFactor, tableName);
            connector.tableOperations().setProperty(tableName, Property.TABLE_FILE_REPLICATION.getKey(), repFactor);
        }
        // Enable Bloom filters using ElementFunctor
        LOGGER.info("Enabling Bloom filter on table {}", tableName);
        connector.tableOperations().setProperty(tableName, Property.TABLE_BLOOM_ENABLED.getKey(), "true");
        connector.tableOperations().setProperty(tableName, Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), store.getKeyPackage().getKeyFunctor().getClass().getName());
        // Remove versioning iterator from table for all scopes
        LOGGER.info("Removing versioning iterator from table {}", tableName);
        final EnumSet<IteratorScope> iteratorScopes = EnumSet.allOf(IteratorScope.class);
        connector.tableOperations().removeIterator(tableName, "vers", iteratorScopes);
        if (store.getSchema().isAggregationEnabled()) {
            // Add Combiner iterator to table for all scopes
            LOGGER.info("Adding Aggregator iterator to table {} for all scopes", tableName);
            connector.tableOperations().attachIterator(tableName, store.getKeyPackage().getIteratorFactory().getAggregatorIteratorSetting(store));
        } else {
            LOGGER.info("Aggregator iterator has not been added to table {}", tableName);
        }
        if (store.getProperties().getEnableValidatorIterator()) {
            // Add validator iterator to table for all scopes
            final IteratorSetting itrSetting = store.getKeyPackage().getIteratorFactory().getValidatorIteratorSetting(store);
            if (null == itrSetting) {
                LOGGER.info("Not adding Validator iterator to table {} as there are no validation functions defined in the schema", tableName);
            } else {
                LOGGER.info("Adding Validator iterator to table {} for all scopes", tableName);
                connector.tableOperations().attachIterator(tableName, store.getKeyPackage().getIteratorFactory().getValidatorIteratorSetting(store));
            }
        } else {
            LOGGER.info("Validator iterator has not been added to table {}", tableName);
        }
    } catch (final AccumuloSecurityException | TableNotFoundException | AccumuloException | IteratorSettingException e) {
        throw new StoreException(e.getMessage(), e);
    }
    setLocalityGroups(store);
}

Example 25 with IteratorSettingException

use of uk.gov.gchq.gaffer.accumulostore.key.exception.IteratorSettingException in project Gaffer by gchq.

the class TableUtils method validateTable.

private static void validateTable(final AccumuloStore store, final String tableName, final Connector connector) throws StoreException {
    final IteratorSetting requiredAggItrSetting;
    if (store.getSchema().isAggregationEnabled()) {
        try {
            requiredAggItrSetting = store.getKeyPackage().getIteratorFactory().getAggregatorIteratorSetting(store);
            if (null != requiredAggItrSetting) {
                requiredAggItrSetting.removeOption(AccumuloStoreConstants.SCHEMA);
                requiredAggItrSetting.removeOption(COLUMN_FAMILIES_OPTION);
            }
        } catch (final IteratorSettingException e) {
            throw new StoreException("Unable to create aggregator iterator settings", e);
        }
    } else {
        requiredAggItrSetting = null;
    }
    final IteratorSetting requiredValidatorItrSetting;
    if (store.getProperties().getEnableValidatorIterator()) {
        requiredValidatorItrSetting = store.getKeyPackage().getIteratorFactory().getValidatorIteratorSetting(store);
        if (null != requiredValidatorItrSetting) {
            requiredValidatorItrSetting.removeOption(AccumuloStoreConstants.SCHEMA);
            requiredValidatorItrSetting.removeOption(COLUMN_FAMILIES_OPTION);
        }
    } else {
        requiredValidatorItrSetting = null;
    }
    final ValidationResult validationResult = new ValidationResult();
    for (final IteratorScope iteratorScope : EnumSet.allOf(IteratorScope.class)) {
        final IteratorSetting aggItrSetting;
        final IteratorSetting validatorItrSetting;
        final IteratorSetting versioningIterSetting;
        try {
            aggItrSetting = store.getConnection().tableOperations().getIteratorSetting(tableName, AccumuloStoreConstants.AGGREGATOR_ITERATOR_NAME, iteratorScope);
            if (null != aggItrSetting) {
                aggItrSetting.removeOption(AccumuloStoreConstants.SCHEMA);
                aggItrSetting.removeOption(COLUMN_FAMILIES_OPTION);
            }
            validatorItrSetting = store.getConnection().tableOperations().getIteratorSetting(tableName, AccumuloStoreConstants.VALIDATOR_ITERATOR_NAME, iteratorScope);
            if (null != validatorItrSetting) {
                validatorItrSetting.removeOption(AccumuloStoreConstants.SCHEMA);
                validatorItrSetting.removeOption(COLUMN_FAMILIES_OPTION);
            }
            versioningIterSetting = store.getConnection().tableOperations().getIteratorSetting(tableName, "vers", iteratorScope);
        } catch (final AccumuloSecurityException | AccumuloException | TableNotFoundException e) {
            throw new StoreException("Unable to find iterators on the table " + tableName, e);
        }
        if (!Objects.equals(requiredAggItrSetting, aggItrSetting)) {
            validationResult.addError("Aggregator iterator for scope " + iteratorScope.name() + " is not as expected. " + "Expected: " + requiredAggItrSetting + ", but found: " + aggItrSetting);
        }
        if (!Objects.equals(requiredValidatorItrSetting, validatorItrSetting)) {
            validationResult.addError("Validator iterator for scope " + iteratorScope.name() + " is not as expected. " + "Expected: " + requiredValidatorItrSetting + ", but found: " + validatorItrSetting);
        }
        if (null != versioningIterSetting) {
            validationResult.addError("The versioning iterator for scope " + iteratorScope.name() + " should not be set on the table.");
        }
    }
    final Iterable<Map.Entry<String, String>> tableProps;
    try {
        tableProps = connector.tableOperations().getProperties(tableName);
    } catch (final AccumuloException | TableNotFoundException e) {
        throw new StoreException("Unable to get table properties.", e);
    }
    boolean bloomFilterEnabled = false;
    String bloomKeyFunctor = null;
    for (final Map.Entry<String, String> tableProp : tableProps) {
        if (Property.TABLE_BLOOM_ENABLED.getKey().equals(tableProp.getKey())) {
            if (Boolean.parseBoolean(tableProp.getValue())) {
                bloomFilterEnabled = true;
            }
        } else if (Property.TABLE_BLOOM_KEY_FUNCTOR.getKey().equals(tableProp.getKey())) {
            if (null == bloomKeyFunctor || CoreKeyBloomFunctor.class.getName().equals(tableProp.getValue())) {
                bloomKeyFunctor = tableProp.getValue();
            }
        }
    }
    if (!bloomFilterEnabled) {
        validationResult.addError("Bloom filter is not enabled. " + Property.TABLE_BLOOM_ENABLED.getKey() + " = " + bloomFilterEnabled);
    }
    if (!CoreKeyBloomFunctor.class.getName().equals(bloomKeyFunctor)) {
        validationResult.addError("Bloom key functor class is incorrect. " + "Expected: " + CoreKeyBloomFunctor.class.getName() + ", but found: " + bloomKeyFunctor);
    }
    if (!validationResult.isValid()) {
        throw new StoreException("Your table " + tableName + " is configured incorrectly. " + validationResult.getErrorString() + "\nEither delete the table and let Gaffer create it for you or fix it manually using the Accumulo shell or the Gaffer AddUpdateTableIterator utility.");
    }
}