Examples with IteratorSettingBuilder uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder used on opensource projects

Example 11 with IteratorSettingBuilder

use of uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder in project Gaffer by gchq.

the class ByteEntityIteratorSettingsFactory method getElementPropertyRangeQueryFilter.

@Override
public IteratorSetting getElementPropertyRangeQueryFilter(final GetElementsOperation<?, ?> operation) {
    final boolean includeEntities = operation.isIncludeEntities();
    final IncludeEdgeType includeEdgeType = operation.getIncludeEdges();
    final IncludeIncomingOutgoingType includeIncomingOutgoingType = operation.getIncludeIncomingOutGoing();
    final boolean deduplicateUndirectedEdges = operation instanceof GetAllElements;
    if (includeEdgeType == IncludeEdgeType.ALL && includeIncomingOutgoingType == IncludeIncomingOutgoingType.BOTH && includeEntities && !deduplicateUndirectedEdges) {
        return null;
    }
    return new IteratorSettingBuilder(AccumuloStoreConstants.RANGE_ELEMENT_PROPERTY_FILTER_ITERATOR_PRIORITY, AccumuloStoreConstants.RANGE_ELEMENT_PROPERTY_FILTER_ITERATOR_NAME, RANGE_ELEMENT_PROPERTY_FILTER_ITERATOR).all().includeIncomingOutgoing(includeIncomingOutgoingType).includeEdges(includeEdgeType).includeEntities(includeEntities).deduplicateUndirectedEdges(deduplicateUndirectedEdges).build();
}

Example 12 with IteratorSettingBuilder

use of uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder in project Gaffer by gchq.

the class ClassicIteratorSettingsFactory method getElementPropertyRangeQueryFilter.

@Override
public IteratorSetting getElementPropertyRangeQueryFilter(final GetElementsOperation<?, ?> operation) {
    final boolean includeEntities = operation.isIncludeEntities();
    final IncludeEdgeType includeEdgeType = operation.getIncludeEdges();
    if (includeEdgeType != IncludeEdgeType.NONE && includeEntities) {
        return null;
    }
    return new IteratorSettingBuilder(AccumuloStoreConstants.RANGE_ELEMENT_PROPERTY_FILTER_ITERATOR_PRIORITY, AccumuloStoreConstants.RANGE_ELEMENT_PROPERTY_FILTER_ITERATOR_NAME, RANGE_ELEMENT_PROPERTY_FILTER_ITERATOR).all().includeEdges(includeEdgeType).includeEntities(includeEntities).build();
}

Example 13 with IteratorSettingBuilder

use of uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder in project Gaffer by gchq.

the class CoreKeyGroupByAggregatorIteratorTest method testAggregatingSinglePropertySet.

public void testAggregatingSinglePropertySet(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException {
    String visibilityString = "public";
    try {
        // Create edge
        final Edge edge = new Edge.Builder().group(TestGroups.EDGE).source("1").dest("2").directed(true).property(AccumuloPropertyNames.COLUMN_QUALIFIER, 8).property(AccumuloPropertyNames.COUNT, 1).build();
        final Properties properties1 = new Properties();
        properties1.put(AccumuloPropertyNames.COUNT, 1);
        // Accumulo key
        final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
        // Accumulo values
        final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties1);
        // Create mutation
        final Mutation m1 = new Mutation(key.getRow());
        m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
        // Write mutation
        final BatchWriterConfig writerConfig = new BatchWriterConfig();
        writerConfig.setMaxMemory(1000000L);
        writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
        writerConfig.setMaxWriteThreads(1);
        final BatchWriter writer = store.getConnection().createBatchWriter(store.getTableName(), writerConfig);
        writer.addMutation(m1);
        writer.close();
        final Edge expectedEdge = new Edge.Builder().group(TestGroups.EDGE).source("1").dest("2").directed(true).property(AccumuloPropertyNames.COLUMN_QUALIFIER, 8).property(AccumuloPropertyNames.COUNT, 1).build();
        // Read data back and check we get one merged element
        final Authorizations authorizations = new Authorizations(visibilityString);
        final Scanner scanner = store.getConnection().createScanner(store.getTableName(), authorizations);
        final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
        scanner.addScanIterator(iteratorSetting);
        final Iterator<Entry<Key, Value>> it = scanner.iterator();
        final Entry<Key, Value> entry = it.next();
        final Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue(), false);
        assertEquals(expectedEdge, readEdge);
        assertEquals(8, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(1, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        // Check no more entries
        if (it.hasNext()) {
            fail("Additional row found.");
        }
    } catch (final AccumuloException | TableNotFoundException e) {
        fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    }
}

Example 14 with IteratorSettingBuilder

use of uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder in project Gaffer by gchq.

the class CoreKeyGroupByAggregatorIteratorTest method testAggregatingEmptyColumnQualifier.

public void testAggregatingEmptyColumnQualifier(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException {
    final String visibilityString = "public";
    try {
        // Create edge
        final Edge edge = new Edge.Builder().group(TestGroups.EDGE).source("1").dest("2").directed(true).property(AccumuloPropertyNames.COLUMN_QUALIFIER, 8).property(AccumuloPropertyNames.PROP_1, 0).property(AccumuloPropertyNames.PROP_2, 0).property(AccumuloPropertyNames.PROP_3, 0).property(AccumuloPropertyNames.PROP_4, 0).property(AccumuloPropertyNames.COUNT, 1).build();
        // THIS EDGE WILL BE REDUCED MEANING ITS CQ (columnQualifier) will only occur once because its key is equal.
        final Edge edge2 = new Edge.Builder().group(TestGroups.EDGE).source("1").dest("2").directed(true).property(AccumuloPropertyNames.PROP_1, 0).property(AccumuloPropertyNames.PROP_2, 0).property(AccumuloPropertyNames.PROP_3, 0).property(AccumuloPropertyNames.PROP_4, 0).property(AccumuloPropertyNames.COUNT, 2).build();
        final Edge edge3 = new Edge.Builder().group(TestGroups.EDGE).source("1").dest("2").directed(true).property(AccumuloPropertyNames.PROP_1, 0).property(AccumuloPropertyNames.PROP_2, 0).property(AccumuloPropertyNames.PROP_3, 0).property(AccumuloPropertyNames.PROP_4, 0).property(AccumuloPropertyNames.COUNT, 10).build();
        // Accumulo key
        final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
        final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
        final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
        // Accumulo values
        final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge.getProperties());
        final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge2.getProperties());
        final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge3.getProperties());
        // Create mutation
        final Mutation m1 = new Mutation(key.getRow());
        m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
        final Mutation m2 = new Mutation(key.getRow());
        m2.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value2);
        final Mutation m3 = new Mutation(key.getRow());
        m3.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value3);
        final Mutation m4 = new Mutation(key2.getRow());
        m4.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value1);
        final Mutation m5 = new Mutation(key.getRow());
        m5.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value1);
        // Write mutation
        final BatchWriterConfig writerConfig = new BatchWriterConfig();
        writerConfig.setMaxMemory(1000000L);
        writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
        writerConfig.setMaxWriteThreads(1);
        final BatchWriter writer = store.getConnection().createBatchWriter(store.getTableName(), writerConfig);
        writer.addMutation(m1);
        writer.addMutation(m2);
        writer.addMutation(m3);
        writer.addMutation(m4);
        writer.addMutation(m5);
        writer.close();
        Edge expectedEdge = new Edge.Builder().group(TestGroups.EDGE).source("1").dest("2").directed(true).property(AccumuloPropertyNames.COLUMN_QUALIFIER, 8).property(AccumuloPropertyNames.COUNT, 15).property(AccumuloPropertyNames.PROP_1, 0).property(AccumuloPropertyNames.PROP_2, 0).property(AccumuloPropertyNames.PROP_3, 0).property(AccumuloPropertyNames.PROP_4, 0).build();
        // Read data back and check we get one merged element
        final Authorizations authorizations = new Authorizations(visibilityString);
        final Scanner scanner = store.getConnection().createScanner(store.getTableName(), authorizations);
        final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
        scanner.addScanIterator(iteratorSetting);
        final Iterator<Entry<Key, Value>> it = scanner.iterator();
        final Entry<Key, Value> entry = it.next();
        final Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue(), false);
        assertEquals(expectedEdge, readEdge);
        assertEquals(8, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(15, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        // Check no more entries
        if (it.hasNext()) {
            fail("Additional row found.");
        }
    } catch (final AccumuloException | TableNotFoundException e) {
        fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    }
}

Example 15 with IteratorSettingBuilder

use of uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder in project Gaffer by gchq.

the class AbstractCoreKeyIteratorSettingsFactory method getElementPostAggregationFilterIteratorSetting.

@Override
public IteratorSetting getElementPostAggregationFilterIteratorSetting(final View view, final AccumuloStore store) throws IteratorSettingException {
    if (!view.hasPostAggregationFilters()) {
        LOGGER.debug("Returning null from getElementPostAggregationFilterIteratorSetting as view.hasPostAggregationFilters = {}", view.hasPostAggregationFilters());
        return null;
    }
    final IteratorSetting is = new IteratorSettingBuilder(AccumuloStoreConstants.ELEMENT_POST_AGGREGATION_FILTER_ITERATOR_PRIORITY, AccumuloStoreConstants.ELEMENT_POST_AGGREGATION_FILTER_ITERATOR_NAME, ElementPostAggregationFilter.class).schema(store.getSchema()).view(view).keyConverter(store.getKeyPackage().getKeyConverter()).build();
    LOGGER.debug("Creating IteratorSetting for iterator class {} with priority = {}, " + "schema = {}, view = {}, keyConverter = {}", ElementPostAggregationFilter.class.getName(), AccumuloStoreConstants.ELEMENT_POST_AGGREGATION_FILTER_ITERATOR_PRIORITY, store.getSchema(), view, store.getKeyPackage().getKeyConverter());
    return is;
}