Examples with Properties uk.gov.gchq.gaffer.data.element.Properties used on opensource projects

Example 26 with Properties

use of uk.gov.gchq.gaffer.data.element.Properties in project Gaffer by gchq.

the class AccumuloKeyValueReducer method reduceMultiValue.

private Value reduceMultiValue(final Key key, final Iterator<Value> iter, final Value firstValue) {
    final String group;
    try {
        group = new String(key.getColumnFamilyData().getBackingArray(), CommonConstants.UTF_8);
    } catch (final UnsupportedEncodingException e) {
        throw new RuntimeException(e.getMessage(), e);
    }
    ElementAggregator aggregator;
    Properties firstPropertySet;
    try {
        firstPropertySet = elementConverter.getPropertiesFromValue(group, firstValue);
        aggregator = schema.getElement(group).getAggregator();
        aggregator.aggregate(firstPropertySet);
        while (iter.hasNext()) {
            aggregator.aggregate(elementConverter.getPropertiesFromValue(group, iter.next()));
        }
    } catch (final AccumuloElementConversionException e) {
        throw new IllegalArgumentException("Failed to get Properties from an accumulo value", e);
    }
    final Properties properties = new Properties();
    aggregator.state(properties);
    try {
        return elementConverter.getValueFromProperties(group, properties);
    } catch (final AccumuloElementConversionException e) {
        throw new IllegalArgumentException("Failed to get Properties from an accumulo value", e);
    }
}

Example 27 with Properties

use of uk.gov.gchq.gaffer.data.element.Properties in project Gaffer by gchq.

the class AbstractAccumuloElementConverterTest method shouldBuildTimestampFromDefaultTime.

@Test
public void shouldBuildTimestampFromDefaultTime() throws AccumuloElementConversionException {
    // Given
    final Properties properties = new Properties() {

        {
            put(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
            put(AccumuloPropertyNames.PROP_1, 2);
        }
    };
    // When
    final long timestamp = converter.buildTimestamp(properties);
    // Then
    assertNotNull(timestamp);
}

Example 28 with Properties

use of uk.gov.gchq.gaffer.data.element.Properties in project Gaffer by gchq.

the class AbstractAccumuloElementConverterTest method shouldSerialiseAndDeSerialiseBetweenPropertyAndValueMissingMiddleProperty.

@Test
public void shouldSerialiseAndDeSerialiseBetweenPropertyAndValueMissingMiddleProperty() throws AccumuloElementConversionException {
    Properties properties = new Properties();
    properties.put(AccumuloPropertyNames.PROP_1, 60);
    properties.put(AccumuloPropertyNames.PROP_3, 299);
    properties.put(AccumuloPropertyNames.PROP_4, 10);
    properties.put(AccumuloPropertyNames.COUNT, 8);
    final Value value = converter.getValueFromProperties(TestGroups.EDGE, properties);
    final Properties deSerialisedProperties = converter.getPropertiesFromValue(TestGroups.EDGE, value);
    assertEquals(60, deSerialisedProperties.get(AccumuloPropertyNames.PROP_1));
    assertEquals(299, deSerialisedProperties.get(AccumuloPropertyNames.PROP_3));
    assertEquals(10, deSerialisedProperties.get(AccumuloPropertyNames.PROP_4));
    assertEquals(8, deSerialisedProperties.get(AccumuloPropertyNames.COUNT));
}

Example 29 with Properties

use of uk.gov.gchq.gaffer.data.element.Properties in project Gaffer by gchq.

the class RowIdAggregatorTest method testAggregatingMultiplePropertySetsAcrossRowIDRange.

private void testAggregatingMultiplePropertySetsAcrossRowIDRange(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException, RangeFactoryException {
    String visibilityString = "public";
    try {
        // Create table
        // (this method creates the table, removes the versioning iterator, and adds the SetOfStatisticsCombiner iterator).
        TableUtils.createTable(store);
        final Properties properties1 = new Properties();
        properties1.put(AccumuloPropertyNames.COUNT, 1);
        final Properties properties2 = new Properties();
        properties2.put(AccumuloPropertyNames.COUNT, 1);
        final Properties properties3 = new Properties();
        properties3.put(AccumuloPropertyNames.COUNT, 2);
        // Create edge
        final Edge edge = new Edge(TestGroups.EDGE);
        edge.setSource("2");
        edge.setDestination("1");
        edge.setDirected(true);
        edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
        final Edge edge2 = new Edge(TestGroups.EDGE);
        edge2.setSource("B");
        edge2.setDestination("Z");
        edge2.setDirected(true);
        edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge2.putProperty(AccumuloPropertyNames.PROP_1, 1);
        edge2.putProperty(AccumuloPropertyNames.PROP_2, 1);
        edge2.putProperty(AccumuloPropertyNames.PROP_3, 1);
        edge2.putProperty(AccumuloPropertyNames.PROP_4, 1);
        final Edge edge3 = new Edge(TestGroups.EDGE);
        edge3.setSource("3");
        edge3.setDestination("8");
        edge3.setDirected(true);
        edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
        final Edge edge6 = new Edge("BasicEdge2");
        edge6.setSource("1");
        edge6.setDestination("5");
        edge6.setDirected(true);
        edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
        edge6.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge6.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge6.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge6.putProperty(AccumuloPropertyNames.PROP_4, 0);
        final Edge edge7 = new Edge("BasicEdge2");
        edge7.setSource("2");
        edge7.setDestination("6");
        edge7.setDirected(true);
        edge7.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge7.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge7.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge7.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge7.putProperty(AccumuloPropertyNames.PROP_4, 0);
        final Edge edge8 = new Edge("BasicEdge2");
        edge8.setSource("4");
        edge8.setDestination("8");
        edge8.setDirected(true);
        edge8.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
        edge8.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge8.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge8.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge8.putProperty(AccumuloPropertyNames.PROP_4, 0);
        final Edge edge9 = new Edge("BasicEdge2");
        edge9.setSource("5");
        edge9.setDestination("9");
        edge9.setDirected(true);
        edge9.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
        edge9.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge9.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge9.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge9.putProperty(AccumuloPropertyNames.PROP_4, 0);
        // Accumulo key
        final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
        final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
        final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
        final Key key4 = elementConverter.getKeysFromEdge(edge6).getFirst();
        final Key key5 = elementConverter.getKeysFromEdge(edge7).getFirst();
        final Key key6 = elementConverter.getKeysFromEdge(edge8).getFirst();
        final Key key7 = elementConverter.getKeysFromEdge(edge9).getFirst();
        // Accumulo values
        final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties1);
        final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties2);
        final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties3);
        final Value value4 = elementConverter.getValueFromProperties(TestGroups.EDGE_2, properties1);
        final Value value5 = elementConverter.getValueFromProperties(TestGroups.EDGE_2, properties2);
        //Create mutation
        final Mutation m1 = new Mutation(key.getRow());
        m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
        final Mutation m2 = new Mutation(key.getRow());
        m2.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value2);
        final Mutation m3 = new Mutation(key.getRow());
        m3.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value3);
        final Mutation m4 = new Mutation(key2.getRow());
        m4.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value1);
        final Mutation m5 = new Mutation(key.getRow());
        m5.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value1);
        final Mutation m6 = new Mutation(key4.getRow());
        m6.put(key4.getColumnFamily(), key4.getColumnQualifier(), new ColumnVisibility(key4.getColumnVisibility()), key4.getTimestamp(), value4);
        final Mutation m7 = new Mutation(key5.getRow());
        m7.put(key5.getColumnFamily(), key5.getColumnQualifier(), new ColumnVisibility(key5.getColumnVisibility()), key5.getTimestamp(), value5);
        final Mutation m8 = new Mutation(key6.getRow());
        m8.put(key6.getColumnFamily(), key6.getColumnQualifier(), new ColumnVisibility(key6.getColumnVisibility()), key6.getTimestamp(), value5);
        final Mutation m9 = new Mutation(key7.getRow());
        m9.put(key7.getColumnFamily(), key7.getColumnQualifier(), new ColumnVisibility(key7.getColumnVisibility()), key7.getTimestamp(), value5);
        // Write mutation
        final BatchWriterConfig writerConfig = new BatchWriterConfig();
        writerConfig.setMaxMemory(1000000L);
        writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
        writerConfig.setMaxWriteThreads(1);
        final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
        writer.addMutation(m1);
        writer.addMutation(m2);
        writer.addMutation(m3);
        writer.addMutation(m4);
        writer.addMutation(m5);
        writer.addMutation(m6);
        writer.addMutation(m7);
        writer.addMutation(m8);
        writer.addMutation(m9);
        writer.close();
        // Read data back and check we get one merged element
        final Authorizations authorizations = new Authorizations(visibilityString);
        final BatchScanner scanner = store.getConnection().createBatchScanner(store.getProperties().getTable(), authorizations, 1000);
        try {
            scanner.addScanIterator(store.getKeyPackage().getIteratorFactory().getRowIDAggregatorIteratorSetting(store, "BasicEdge2"));
        } catch (IteratorSettingException e) {
            fail(e.getMessage());
        }
        final RangeFactory rangeF = store.getKeyPackage().getRangeFactory();
        final Range r = rangeF.getRangeFromPair(new Pair<ElementSeed>((new EntitySeed("1")), new EntitySeed("4")), new SummariseGroupOverRanges());
        final Range r2 = rangeF.getRangeFromPair(new Pair<ElementSeed>((new EntitySeed("5")), new EntitySeed("5")), new SummariseGroupOverRanges());
        scanner.setRanges(Arrays.asList(r, r2));
        final Iterator<Entry<Key, Value>> it = scanner.iterator();
        Entry<Key, Value> entry = it.next();
        Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        Edge expectedEdge = new Edge("BasicEdge2");
        expectedEdge.setSource("4");
        expectedEdge.setDestination("8");
        expectedEdge.setDirected(true);
        expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 5);
        expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 3);
        assertEquals(expectedEdge, readEdge);
        assertEquals(5, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(3, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        // Check we get the Result of the second provided range
        assertTrue(it.hasNext());
        entry = it.next();
        readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        expectedEdge = new Edge("BasicEdge2");
        expectedEdge.setSource("5");
        expectedEdge.setDestination("9");
        expectedEdge.setDirected(true);
        expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
        expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 1);
        assertEquals(expectedEdge, readEdge);
        //Check no additional rows are found. (For a table of this size we shouldn't see this)
        if (it.hasNext()) {
            fail("Additional row found.");
        }
    } catch (AccumuloException | TableExistsException | TableNotFoundException e) {
        fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    }
}

Example 30 with Properties

use of uk.gov.gchq.gaffer.data.element.Properties in project Gaffer by gchq.

the class CoreKeyGroupByAggregatorIteratorTest method testAggregatingSinglePropertySet.

public void testAggregatingSinglePropertySet(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
    String visibilityString = "public";
    try {
        // Create edge
        final Edge edge = new Edge(TestGroups.EDGE);
        edge.setSource("1");
        edge.setDestination("2");
        edge.setDirected(true);
        edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
        edge.putProperty(AccumuloPropertyNames.COUNT, 1);
        final Properties properties1 = new Properties();
        properties1.put(AccumuloPropertyNames.COUNT, 1);
        // Accumulo key
        final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
        // Accumulo values
        final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties1);
        // Create mutation
        final Mutation m1 = new Mutation(key.getRow());
        m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
        // Write mutation
        final BatchWriterConfig writerConfig = new BatchWriterConfig();
        writerConfig.setMaxMemory(1000000L);
        writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
        writerConfig.setMaxWriteThreads(1);
        final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
        writer.addMutation(m1);
        writer.close();
        final Edge expectedEdge = new Edge(TestGroups.EDGE);
        expectedEdge.setSource("1");
        expectedEdge.setDestination("2");
        expectedEdge.setDirected(true);
        expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
        expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 1);
        // Read data back and check we get one merged element
        final Authorizations authorizations = new Authorizations(visibilityString);
        final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
        final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
        scanner.addScanIterator(iteratorSetting);
        final Iterator<Entry<Key, Value>> it = scanner.iterator();
        final Entry<Key, Value> entry = it.next();
        final Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        assertEquals(expectedEdge, readEdge);
        assertEquals(8, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(1, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        // Check no more entries
        if (it.hasNext()) {
            fail("Additional row found.");
        }
    } catch (AccumuloException | TableNotFoundException e) {
        fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    }
}