Search in sources :

Example 11 with Properties

use of uk.gov.gchq.gaffer.data.element.Properties in project Gaffer by gchq.

the class AbstractCoreKeyAccumuloElementConverter method getPropertiesFromValue.

@Override
public Properties getPropertiesFromValue(final String group, final Value value) throws AccumuloElementConversionException {
    final Properties properties = new Properties();
    if (value == null || value.getSize() == 0) {
        return properties;
    }
    final byte[] bytes = value.get();
    int lastDelimiter = 0;
    final int arrayLength = bytes.length;
    long currentPropLength;
    final SchemaElementDefinition elementDefinition = schema.getElement(group);
    if (null == elementDefinition) {
        throw new AccumuloElementConversionException("No SchemaElementDefinition found for group " + group + ", is this group in your schema or do your table iterators need updating?");
    }
    final Iterator<String> propertyNames = elementDefinition.getProperties().iterator();
    while (propertyNames.hasNext() && lastDelimiter < arrayLength) {
        final String propertyName = propertyNames.next();
        if (isStoredInValue(propertyName, elementDefinition)) {
            final TypeDefinition typeDefinition = elementDefinition.getPropertyTypeDef(propertyName);
            final Serialisation<?> serialiser = (typeDefinition != null) ? typeDefinition.getSerialiser() : null;
            if (null != serialiser) {
                final int numBytesForLength = CompactRawSerialisationUtils.decodeVIntSize(bytes[lastDelimiter]);
                final byte[] length = new byte[numBytesForLength];
                System.arraycopy(bytes, lastDelimiter, length, 0, numBytesForLength);
                try {
                    currentPropLength = CompactRawSerialisationUtils.readLong(length);
                } catch (final SerialisationException e) {
                    throw new AccumuloElementConversionException("Exception reading length of property", e);
                }
                lastDelimiter += numBytesForLength;
                if (currentPropLength > 0) {
                    try {
                        properties.put(propertyName, serialiser.deserialise(Arrays.copyOfRange(bytes, lastDelimiter, lastDelimiter += currentPropLength)));
                    } catch (SerialisationException e) {
                        throw new AccumuloElementConversionException("Failed to deserialise property " + propertyName, e);
                    }
                } else {
                    try {
                        properties.put(propertyName, serialiser.deserialiseEmptyBytes());
                    } catch (SerialisationException e) {
                        throw new AccumuloElementConversionException("Failed to deserialise property " + propertyName, e);
                    }
                }
            }
        }
    }
    return properties;
}
Also used : SerialisationException(uk.gov.gchq.gaffer.exception.SerialisationException) Properties(uk.gov.gchq.gaffer.data.element.Properties) SchemaElementDefinition(uk.gov.gchq.gaffer.store.schema.SchemaElementDefinition) AccumuloElementConversionException(uk.gov.gchq.gaffer.accumulostore.key.exception.AccumuloElementConversionException) TypeDefinition(uk.gov.gchq.gaffer.store.schema.TypeDefinition)

Example 12 with Properties

use of uk.gov.gchq.gaffer.data.element.Properties in project Gaffer by gchq.

the class AbstractCoreKeyAccumuloElementConverter method getPropertiesFromColumnQualifier.

@Override
public Properties getPropertiesFromColumnQualifier(final String group, final byte[] bytes) throws AccumuloElementConversionException {
    final SchemaElementDefinition elementDefinition = schema.getElement(group);
    if (null == elementDefinition) {
        throw new AccumuloElementConversionException("No SchemaElementDefinition found for group " + group + ", is this group in your schema or do your table iterators need updating?");
    }
    final Properties properties = new Properties();
    if (bytes == null || bytes.length == 0) {
        return properties;
    }
    int lastDelimiter = 0;
    final int arrayLength = bytes.length;
    long currentPropLength;
    final Iterator<String> propertyNames = elementDefinition.getGroupBy().iterator();
    while (propertyNames.hasNext() && lastDelimiter < arrayLength) {
        final String propertyName = propertyNames.next();
        final TypeDefinition typeDefinition = elementDefinition.getPropertyTypeDef(propertyName);
        final Serialisation<?> serialiser = (typeDefinition != null) ? typeDefinition.getSerialiser() : null;
        if (null != serialiser) {
            final int numBytesForLength = CompactRawSerialisationUtils.decodeVIntSize(bytes[lastDelimiter]);
            final byte[] length = new byte[numBytesForLength];
            System.arraycopy(bytes, lastDelimiter, length, 0, numBytesForLength);
            try {
                currentPropLength = CompactRawSerialisationUtils.readLong(length);
            } catch (final SerialisationException e) {
                throw new AccumuloElementConversionException("Exception reading length of property", e);
            }
            lastDelimiter += numBytesForLength;
            if (currentPropLength > 0) {
                try {
                    properties.put(propertyName, serialiser.deserialise(Arrays.copyOfRange(bytes, lastDelimiter, lastDelimiter += currentPropLength)));
                } catch (SerialisationException e) {
                    throw new AccumuloElementConversionException("Failed to deserialise property " + propertyName, e);
                }
            }
        }
    }
    return properties;
}
Also used : SerialisationException(uk.gov.gchq.gaffer.exception.SerialisationException) Properties(uk.gov.gchq.gaffer.data.element.Properties) SchemaElementDefinition(uk.gov.gchq.gaffer.store.schema.SchemaElementDefinition) AccumuloElementConversionException(uk.gov.gchq.gaffer.accumulostore.key.exception.AccumuloElementConversionException) TypeDefinition(uk.gov.gchq.gaffer.store.schema.TypeDefinition)

Example 13 with Properties

use of uk.gov.gchq.gaffer.data.element.Properties in project Gaffer by gchq.

the class RowIDAggregator method reduce.

private Properties reduce(final Iterator<Properties> iter) {
    Properties properties;
    while (iter.hasNext()) {
        properties = iter.next();
        if (properties != null) {
            aggregator.aggregate(properties);
        }
    }
    final Properties result = new Properties();
    aggregator.state(result);
    return result;
}
Also used : Properties(uk.gov.gchq.gaffer.data.element.Properties)

Example 14 with Properties

use of uk.gov.gchq.gaffer.data.element.Properties in project Gaffer by gchq.

the class RowIDAggregator method findTop.

/**
     * Given the current position in the source}, filter to only the columns specified. Sets topKey and topValue to non-null on success
     *
     * @throws IOException Failure to seek
     */
protected void findTop() throws IOException {
    if (!source.hasTop()) {
        return;
    }
    PropertiesIterator iter = new PropertiesIterator(source, currentRange, currentColumnFamilies, currentColumnFamiliesInclusive, group, workKey, elementConverter);
    Properties topProperties = reduce(iter);
    try {
        topValue = elementConverter.getValueFromProperties(group, topProperties);
        topKey = new Key(workKey.getRowData().getBackingArray(), group.getBytes(CommonConstants.UTF_8), elementConverter.buildColumnQualifier(group, topProperties), elementConverter.buildColumnVisibility(group, topProperties), elementConverter.buildTimestamp(topProperties));
    } catch (AccumuloElementConversionException e) {
        throw new RuntimeException(e);
    }
}
Also used : Properties(uk.gov.gchq.gaffer.data.element.Properties) Key(org.apache.accumulo.core.data.Key) PartialKey(org.apache.accumulo.core.data.PartialKey) AccumuloElementConversionException(uk.gov.gchq.gaffer.accumulostore.key.exception.AccumuloElementConversionException)

Example 15 with Properties

use of uk.gov.gchq.gaffer.data.element.Properties in project Gaffer by gchq.

the class GetElementsIT method shouldGetElements.

private void shouldGetElements(final List<Element> expectedElements, final SeedMatchingType seedMatching, final IncludeEdgeType includeEdgeType, final Boolean includeEntities, final IncludeIncomingOutgoingType inOutType, final Iterable<ElementSeed> seeds) throws IOException, OperationException {
    // Given
    final User user = new User();
    final GetElements<ElementSeed, Element> op = new GetElements.Builder<>().seedMatching(seedMatching).build();
    op.setSeeds(seeds);
    op.setIncludeEntities(includeEntities);
    op.setIncludeEdges(includeEdgeType);
    op.setIncludeIncomingOutGoing(inOutType);
    op.setView(new View.Builder().entity(TestGroups.ENTITY).edge(TestGroups.EDGE).build());
    // When
    final CloseableIterable<? extends Element> results = graph.execute(op, user);
    // Then
    final List<Element> expectedElementsCopy = Lists.newArrayList(expectedElements);
    for (final Element result : results) {
        final ElementSeed seed = ElementSeed.createSeed(result);
        if (result instanceof Entity) {
            Entity entity = (Entity) result;
            final Collection<Element> listOfElements = new LinkedList<>();
            Iterables.addAll(listOfElements, results);
            assertTrue("Entity was not expected: " + entity, expectedElements.contains(entity));
        } else {
            Edge edge = (Edge) result;
            if (edge.isDirected()) {
                assertTrue("Edge was not expected: " + edge, expectedElements.contains(edge));
            } else {
                final Edge edgeReversed = new Edge(TestGroups.EDGE, edge.getDestination(), edge.getSource(), edge.isDirected());
                Properties properties = edge.getProperties();
                edgeReversed.copyProperties(properties);
                expectedElementsCopy.remove(edgeReversed);
                assertTrue("Edge was not expected: " + seed, expectedElements.contains(result) || expectedElements.contains(edgeReversed));
            }
        }
        expectedElementsCopy.remove(result);
    }
    assertEquals("The number of elements returned was not as expected. Missing elements: " + expectedElementsCopy + ". Seeds: " + seeds, expectedElements.size(), Lists.newArrayList(results).size());
    assertEquals(new HashSet<>(expectedElements), Sets.newHashSet(results));
}
Also used : Entity(uk.gov.gchq.gaffer.data.element.Entity) User(uk.gov.gchq.gaffer.user.User) Element(uk.gov.gchq.gaffer.data.element.Element) Properties(uk.gov.gchq.gaffer.data.element.Properties) LinkedList(java.util.LinkedList) ElementSeed(uk.gov.gchq.gaffer.operation.data.ElementSeed) Edge(uk.gov.gchq.gaffer.data.element.Edge)

Aggregations

Properties (uk.gov.gchq.gaffer.data.element.Properties)31 Test (org.junit.Test)16 Value (org.apache.accumulo.core.data.Value)11 AccumuloElementConversionException (uk.gov.gchq.gaffer.accumulostore.key.exception.AccumuloElementConversionException)8 Key (org.apache.accumulo.core.data.Key)7 Edge (uk.gov.gchq.gaffer.data.element.Edge)6 Schema (uk.gov.gchq.gaffer.store.schema.Schema)5 Entity (uk.gov.gchq.gaffer.data.element.Entity)4 SchemaElementDefinition (uk.gov.gchq.gaffer.store.schema.SchemaElementDefinition)4 Element (uk.gov.gchq.gaffer.data.element.Element)3 ElementAggregator (uk.gov.gchq.gaffer.data.element.function.ElementAggregator)3 File (java.io.File)2 UnsupportedEncodingException (java.io.UnsupportedEncodingException)2 ArrayList (java.util.ArrayList)2 HashSet (java.util.HashSet)2 Entry (java.util.Map.Entry)2 Random (java.util.Random)2 AccumuloException (org.apache.accumulo.core.client.AccumuloException)2 BatchWriter (org.apache.accumulo.core.client.BatchWriter)2 BatchWriterConfig (org.apache.accumulo.core.client.BatchWriterConfig)2