Search in sources :

Example 26 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.

the class TableUtilsTest method shouldCreateTableCorrectlyIfSchemaContainsNoAggregators.

@Test
public void shouldCreateTableCorrectlyIfSchemaContainsNoAggregators() throws Exception {
    // Given
    final MockAccumuloStore store = new MockAccumuloStore();
    final Schema schema = new Schema.Builder().type(TestTypes.ID_STRING, String.class).type(TestTypes.DIRECTED_TRUE, Boolean.class).edge(TestGroups.EDGE, new SchemaEdgeDefinition.Builder().source(TestTypes.ID_STRING).destination(TestTypes.ID_STRING).directed(TestTypes.DIRECTED_TRUE).build()).build();
    final AccumuloProperties props = AccumuloProperties.loadStoreProperties(StreamUtil.storeProps(TableUtilsTest.class));
    props.setTable(NO_AGGREGATORS_TABLE_NAME);
    store.initialise(schema, props);
    // When
    TableUtils.createTable(store);
    // Then
    final Map<String, EnumSet<IteratorScope>> itrs = store.getConnection().tableOperations().listIterators(NO_AGGREGATORS_TABLE_NAME);
    assertEquals(1, itrs.size());
    final EnumSet<IteratorScope> validator = itrs.get(AccumuloStoreConstants.VALIDATOR_ITERATOR_NAME);
    assertEquals(EnumSet.allOf(IteratorScope.class), validator);
    final IteratorSetting validatorSetting = store.getConnection().tableOperations().getIteratorSetting(NO_AGGREGATORS_TABLE_NAME, AccumuloStoreConstants.VALIDATOR_ITERATOR_NAME, IteratorScope.majc);
    assertEquals(AccumuloStoreConstants.VALIDATOR_ITERATOR_PRIORITY, validatorSetting.getPriority());
    assertEquals(ValidatorFilter.class.getName(), validatorSetting.getIteratorClass());
    final Map<String, String> validatorOptions = validatorSetting.getOptions();
    assertNotNull(Schema.fromJson(validatorOptions.get(AccumuloStoreConstants.SCHEMA).getBytes(CommonConstants.UTF_8)).getEdge(TestGroups.EDGE));
    assertEquals(ByteEntityAccumuloElementConverter.class.getName(), validatorOptions.get(AccumuloStoreConstants.ACCUMULO_ELEMENT_CONVERTER_CLASS));
    final EnumSet<IteratorScope> aggregator = itrs.get(AccumuloStoreConstants.AGGREGATOR_ITERATOR_NAME);
    assertNull(aggregator);
    final IteratorSetting aggregatorSetting = store.getConnection().tableOperations().getIteratorSetting(NO_AGGREGATORS_TABLE_NAME, AccumuloStoreConstants.AGGREGATOR_ITERATOR_NAME, IteratorScope.majc);
    assertNull(aggregatorSetting);
    final Map<String, String> tableProps = new HashMap<>();
    for (final Map.Entry<String, String> entry : store.getConnection().tableOperations().getProperties(NO_AGGREGATORS_TABLE_NAME)) {
        tableProps.put(entry.getKey(), entry.getValue());
    }
    assertEquals(0, Integer.parseInt(tableProps.get(Property.TABLE_FILE_REPLICATION.getKey())));
}
Also used : MockAccumuloStore(uk.gov.gchq.gaffer.accumulostore.MockAccumuloStore) SingleUseMockAccumuloStore(uk.gov.gchq.gaffer.accumulostore.SingleUseMockAccumuloStore) AccumuloProperties(uk.gov.gchq.gaffer.accumulostore.AccumuloProperties) HashMap(java.util.HashMap) Schema(uk.gov.gchq.gaffer.store.schema.Schema) EnumSet(java.util.EnumSet) ValidatorFilter(uk.gov.gchq.gaffer.accumulostore.key.impl.ValidatorFilter) IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) ByteEntityAccumuloElementConverter(uk.gov.gchq.gaffer.accumulostore.key.core.impl.byteEntity.ByteEntityAccumuloElementConverter) IteratorScope(org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope) HashMap(java.util.HashMap) Map(java.util.Map) Test(org.junit.Test)

Example 27 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.

the class CoreKeyGroupByAggregatorIteratorTest method testAggregatingEmptyColumnQualifier.

public void testAggregatingEmptyColumnQualifier(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
    final String visibilityString = "public";
    try {
        // Create edge
        final Edge edge = new Edge(TestGroups.EDGE);
        edge.setSource("1");
        edge.setDestination("2");
        edge.setDirected(true);
        edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
        edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge.putProperty(AccumuloPropertyNames.COUNT, 1);
        //THIS EDGE WILL BE REDUCED MEANING ITS CQ (columnQualifier) will only occur once because its key is equal.
        final Edge edge2 = new Edge(TestGroups.EDGE);
        edge2.setSource("1");
        edge2.setDestination("2");
        edge2.setDirected(true);
        edge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge2.putProperty(AccumuloPropertyNames.COUNT, 2);
        final Edge edge3 = new Edge(TestGroups.EDGE);
        edge3.setSource("1");
        edge3.setDestination("2");
        edge3.setDirected(true);
        edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge3.putProperty(AccumuloPropertyNames.COUNT, 10);
        // Accumulo key
        final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
        final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
        final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
        // Accumulo values
        final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge.getProperties());
        final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge2.getProperties());
        final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge3.getProperties());
        // Create mutation
        final Mutation m1 = new Mutation(key.getRow());
        m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
        final Mutation m2 = new Mutation(key.getRow());
        m2.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value2);
        final Mutation m3 = new Mutation(key.getRow());
        m3.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value3);
        final Mutation m4 = new Mutation(key2.getRow());
        m4.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value1);
        final Mutation m5 = new Mutation(key.getRow());
        m5.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value1);
        // Write mutation
        final BatchWriterConfig writerConfig = new BatchWriterConfig();
        writerConfig.setMaxMemory(1000000L);
        writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
        writerConfig.setMaxWriteThreads(1);
        final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
        writer.addMutation(m1);
        writer.addMutation(m2);
        writer.addMutation(m3);
        writer.addMutation(m4);
        writer.addMutation(m5);
        writer.close();
        Edge expectedEdge = new Edge(TestGroups.EDGE);
        expectedEdge.setSource("1");
        expectedEdge.setDestination("2");
        expectedEdge.setDirected(true);
        expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
        expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 15);
        expectedEdge.putProperty(AccumuloPropertyNames.PROP_1, 0);
        expectedEdge.putProperty(AccumuloPropertyNames.PROP_2, 0);
        expectedEdge.putProperty(AccumuloPropertyNames.PROP_3, 0);
        expectedEdge.putProperty(AccumuloPropertyNames.PROP_4, 0);
        // Read data back and check we get one merged element
        final Authorizations authorizations = new Authorizations(visibilityString);
        final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
        final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
        scanner.addScanIterator(iteratorSetting);
        final Iterator<Entry<Key, Value>> it = scanner.iterator();
        final Entry<Key, Value> entry = it.next();
        final Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        assertEquals(expectedEdge, readEdge);
        assertEquals(8, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(15, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        // Check no more entries
        if (it.hasNext()) {
            fail("Additional row found.");
        }
    } catch (AccumuloException | TableNotFoundException e) {
        fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    }
}
Also used : Scanner(org.apache.accumulo.core.client.Scanner) AccumuloException(org.apache.accumulo.core.client.AccumuloException) Authorizations(org.apache.accumulo.core.security.Authorizations) IteratorSettingBuilder(uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder) Element(uk.gov.gchq.gaffer.data.element.Element) IteratorSettingBuilder(uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder) TableNotFoundException(org.apache.accumulo.core.client.TableNotFoundException) Entry(java.util.Map.Entry) IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) Value(org.apache.accumulo.core.data.Value) BatchWriterConfig(org.apache.accumulo.core.client.BatchWriterConfig) Mutation(org.apache.accumulo.core.data.Mutation) ColumnVisibility(org.apache.accumulo.core.security.ColumnVisibility) BatchWriter(org.apache.accumulo.core.client.BatchWriter) Edge(uk.gov.gchq.gaffer.data.element.Edge) Key(org.apache.accumulo.core.data.Key)

Example 28 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.

the class CoreKeyGroupByAggregatorIteratorTest method testAggregatingSinglePropertySet.

public void testAggregatingSinglePropertySet(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
    String visibilityString = "public";
    try {
        // Create edge
        final Edge edge = new Edge(TestGroups.EDGE);
        edge.setSource("1");
        edge.setDestination("2");
        edge.setDirected(true);
        edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
        edge.putProperty(AccumuloPropertyNames.COUNT, 1);
        final Properties properties1 = new Properties();
        properties1.put(AccumuloPropertyNames.COUNT, 1);
        // Accumulo key
        final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
        // Accumulo values
        final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties1);
        // Create mutation
        final Mutation m1 = new Mutation(key.getRow());
        m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
        // Write mutation
        final BatchWriterConfig writerConfig = new BatchWriterConfig();
        writerConfig.setMaxMemory(1000000L);
        writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
        writerConfig.setMaxWriteThreads(1);
        final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
        writer.addMutation(m1);
        writer.close();
        final Edge expectedEdge = new Edge(TestGroups.EDGE);
        expectedEdge.setSource("1");
        expectedEdge.setDestination("2");
        expectedEdge.setDirected(true);
        expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
        expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 1);
        // Read data back and check we get one merged element
        final Authorizations authorizations = new Authorizations(visibilityString);
        final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
        final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
        scanner.addScanIterator(iteratorSetting);
        final Iterator<Entry<Key, Value>> it = scanner.iterator();
        final Entry<Key, Value> entry = it.next();
        final Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        assertEquals(expectedEdge, readEdge);
        assertEquals(8, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(1, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        // Check no more entries
        if (it.hasNext()) {
            fail("Additional row found.");
        }
    } catch (AccumuloException | TableNotFoundException e) {
        fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    }
}
Also used : Scanner(org.apache.accumulo.core.client.Scanner) AccumuloException(org.apache.accumulo.core.client.AccumuloException) Authorizations(org.apache.accumulo.core.security.Authorizations) IteratorSettingBuilder(uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder) Element(uk.gov.gchq.gaffer.data.element.Element) Properties(uk.gov.gchq.gaffer.data.element.Properties) AccumuloProperties(uk.gov.gchq.gaffer.accumulostore.AccumuloProperties) IteratorSettingBuilder(uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder) TableNotFoundException(org.apache.accumulo.core.client.TableNotFoundException) Entry(java.util.Map.Entry) IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) Value(org.apache.accumulo.core.data.Value) BatchWriterConfig(org.apache.accumulo.core.client.BatchWriterConfig) Mutation(org.apache.accumulo.core.data.Mutation) ColumnVisibility(org.apache.accumulo.core.security.ColumnVisibility) BatchWriter(org.apache.accumulo.core.client.BatchWriter) Edge(uk.gov.gchq.gaffer.data.element.Edge) Key(org.apache.accumulo.core.data.Key)

Example 29 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.

the class AccumuloStore method updateConfiguration.

/**
     * Updates a Hadoop {@link Configuration} with information needed to connect to the Accumulo store. It adds
     * iterators to apply the provided {@link View}. This method will be used by operations that run MapReduce
     * or Spark jobs against the Accumulo store.
     *
     * @param conf A {@link Configuration} to be updated.
     * @param view The {@link View} to be applied.
     * @param user The {@link User} to be used.
     * @throws StoreException if there is a failure to connect to Accumulo or a problem setting the iterators.
     */
public void updateConfiguration(final Configuration conf, final View view, final User user) throws StoreException {
    try {
        // Table name
        InputConfigurator.setInputTableName(AccumuloInputFormat.class, conf, getProperties().getTable());
        // User
        addUserToConfiguration(conf);
        // Authorizations
        Authorizations authorisations;
        if (null != user && null != user.getDataAuths()) {
            authorisations = new Authorizations(user.getDataAuths().toArray(new String[user.getDataAuths().size()]));
        } else {
            authorisations = new Authorizations();
        }
        InputConfigurator.setScanAuthorizations(AccumuloInputFormat.class, conf, authorisations);
        // Zookeeper
        addZookeeperToConfiguration(conf);
        // Add keypackage, schema and view to conf
        conf.set(ElementInputFormat.KEY_PACKAGE, getProperties().getKeyPackageClass());
        conf.set(ElementInputFormat.SCHEMA, new String(getSchema().toCompactJson(), CommonConstants.UTF_8));
        conf.set(ElementInputFormat.VIEW, new String(view.toCompactJson(), CommonConstants.UTF_8));
        // Add iterators that depend on the view
        if (view.hasGroups()) {
            IteratorSetting elementPreFilter = getKeyPackage().getIteratorFactory().getElementPreAggregationFilterIteratorSetting(view, this);
            IteratorSetting elementPostFilter = getKeyPackage().getIteratorFactory().getElementPostAggregationFilterIteratorSetting(view, this);
            InputConfigurator.addIterator(AccumuloInputFormat.class, conf, elementPostFilter);
            InputConfigurator.addIterator(AccumuloInputFormat.class, conf, elementPreFilter);
        }
    } catch (final AccumuloSecurityException | IteratorSettingException | UnsupportedEncodingException e) {
        throw new StoreException(e);
    }
}
Also used : Authorizations(org.apache.accumulo.core.security.Authorizations) IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) UnsupportedEncodingException(java.io.UnsupportedEncodingException) AccumuloSecurityException(org.apache.accumulo.core.client.AccumuloSecurityException) IteratorSettingException(uk.gov.gchq.gaffer.accumulostore.key.exception.IteratorSettingException) StoreException(uk.gov.gchq.gaffer.store.StoreException)

Example 30 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.

the class AccumuloRetriever method getScanner.

/**
     * Create a scanner to use used in your query.
     * <p>
     *
     * @param ranges the ranges to get the scanner for
     * @return A {@link org.apache.accumulo.core.client.BatchScanner} for the
     * table specified in the properties with the ranges provided.
     * @throws TableNotFoundException if an accumulo table could not be found
     * @throws StoreException         if a connection to accumulo could not be created.
     */
protected BatchScanner getScanner(final Set<Range> ranges) throws TableNotFoundException, StoreException {
    final BatchScanner scanner = store.getConnection().createBatchScanner(store.getProperties().getTable(), authorisations, store.getProperties().getThreadsForBatchScanner());
    if (iteratorSettings != null) {
        for (final IteratorSetting iteratorSetting : iteratorSettings) {
            if (iteratorSetting != null) {
                scanner.addScanIterator(iteratorSetting);
            }
        }
    }
    scanner.setRanges(ranges);
    // Currently hard links element class to column family position.
    if (IncludeEdgeType.NONE != operation.getIncludeEdges()) {
        for (final String col : operation.getView().getEdgeGroups()) {
            scanner.fetchColumnFamily(new Text(col));
        }
    }
    if (operation.isIncludeEntities()) {
        for (final String col : operation.getView().getEntityGroups()) {
            scanner.fetchColumnFamily(new Text(col));
        }
    }
    return scanner;
}
Also used : IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) BatchScanner(org.apache.accumulo.core.client.BatchScanner) Text(org.apache.hadoop.io.Text)

Aggregations

IteratorSetting (org.apache.accumulo.core.client.IteratorSetting)35 Test (org.junit.Test)16 Scanner (org.apache.accumulo.core.client.Scanner)13 Authorizations (org.apache.accumulo.core.security.Authorizations)13 Key (org.apache.accumulo.core.data.Key)11 Value (org.apache.accumulo.core.data.Value)11 PasswordToken (org.apache.accumulo.core.client.security.tokens.PasswordToken)10 ColumnMapper (org.apache.hadoop.hive.accumulo.columns.ColumnMapper)10 Entry (java.util.Map.Entry)9 BatchWriterConfig (org.apache.accumulo.core.client.BatchWriterConfig)8 Range (org.apache.accumulo.core.data.Range)8 Map (java.util.Map)7 AccumuloException (org.apache.accumulo.core.client.AccumuloException)7 TableNotFoundException (org.apache.accumulo.core.client.TableNotFoundException)7 Mutation (org.apache.accumulo.core.data.Mutation)7 ColumnVisibility (org.apache.accumulo.core.security.ColumnVisibility)7 AccumuloConnectionParameters (org.apache.hadoop.hive.accumulo.AccumuloConnectionParameters)7 BatchWriter (org.apache.accumulo.core.client.BatchWriter)6 IteratorSettingBuilder (uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder)6 Edge (uk.gov.gchq.gaffer.data.element.Edge)6