Examples with TypeDefinition uk.gov.gchq.gaffer.store.schema.TypeDefinition used on opensource projects

Example 11 with TypeDefinition

use of uk.gov.gchq.gaffer.store.schema.TypeDefinition in project Gaffer by gchq.

the class SchemaUtils method buildParquetSchema.

private MessageType buildParquetSchema(final String group) throws SerialisationException {
    SchemaElementDefinition groupGafferSchema;
    final boolean isEntity = gafferSchema.getEntityGroups().contains(group);
    final StringBuilder schemaString = new StringBuilder("message Element {\n");
    Serialiser serialiser = gafferSchema.getVertexSerialiser();
    // Check that the vertex does not get stored as nested data
    if (serialiser instanceof ParquetSerialiser && ((ParquetSerialiser) serialiser).getParquetSchema("test").contains(" group ")) {
        throw new SerialisationException("Can not have a vertex that is serialised as nested data as it can not be indexed");
    }
    if (isEntity) {
        groupGafferSchema = gafferSchema.getEntity(group);
        schemaString.append(convertColumnSerialiserToParquetColumns(serialiser, ParquetStore.VERTEX)).append("\n");
        addGroupColumnToSerialiser(group, ParquetStore.VERTEX, serialiser);
    } else {
        groupGafferSchema = gafferSchema.getEdge(group);
        schemaString.append(convertColumnSerialiserToParquetColumns(serialiser, ParquetStore.SOURCE)).append("\n");
        addGroupColumnToSerialiser(group, ParquetStore.SOURCE, serialiser);
        schemaString.append(convertColumnSerialiserToParquetColumns(serialiser, ParquetStore.DESTINATION)).append("\n");
        addGroupColumnToSerialiser(group, ParquetStore.DESTINATION, serialiser);
        addGroupColumnToSerialiser(group, ParquetStore.DIRECTED, BooleanParquetSerialiser.class.getCanonicalName());
        schemaString.append(convertColumnSerialiserToParquetColumns(getSerialiser(BooleanParquetSerialiser.class.getCanonicalName()), ParquetStore.DIRECTED)).append("\n");
    }
    Map<String, String> propertyMap = groupGafferSchema.getPropertyMap();
    for (final Map.Entry<String, String> entry : propertyMap.entrySet()) {
        if (entry.getKey().contains("_") || entry.getKey().contains(".")) {
            throw new SchemaException("The ParquetStore does not support properties which contain the characters '_' or '.'");
        }
        final TypeDefinition type = gafferSchema.getType(entry.getValue());
        addGroupColumnToSerialiser(group, entry.getKey(), type.getSerialiserClass());
        schemaString.append(convertColumnSerialiserToParquetColumns(getSerialiser(type.getSerialiserClass()), entry.getKey())).append("\n");
    }
    schemaString.append("}");
    String parquetSchemaString = schemaString.toString();
    final MessageType parquetSchema = MessageTypeParser.parseMessageType(parquetSchemaString);
    LOGGER.debug("Generated Parquet schema: " + parquetSchemaString);
    return parquetSchema;
}

Example 12 with TypeDefinition

use of uk.gov.gchq.gaffer.store.schema.TypeDefinition in project Gaffer by gchq.

the class PropertiesSerialiser method serialiseProperties.

protected void serialiseProperties(final Properties properties, final SchemaElementDefinition elementDefinition, final ByteArrayOutputStream out) throws SerialisationException {
    for (final String propertyName : elementDefinition.getProperties()) {
        final TypeDefinition typeDefinition = elementDefinition.getPropertyTypeDef(propertyName);
        final ToBytesSerialiser<Object> serialiser = (null != typeDefinition) ? (ToBytesSerialiser) typeDefinition.getSerialiser() : null;
        LengthValueBytesSerialiserUtil.serialise(serialiser, properties.get(propertyName), out);
    }
}

Example 13 with TypeDefinition

use of uk.gov.gchq.gaffer.store.schema.TypeDefinition in project Gaffer by gchq.

the class PropertiesSerialiser method deserialiseProperties.

protected void deserialiseProperties(final byte[] bytes, final Properties properties, final SchemaElementDefinition elementDefinition, final int[] delimiter) throws SerialisationException {
    final int arrayLength = bytes.length;
    final Iterator<String> propertyNames = elementDefinition.getProperties().iterator();
    while (propertyNames.hasNext() && delimiter[0] < arrayLength) {
        final String propertyName = propertyNames.next();
        final TypeDefinition typeDefinition = elementDefinition.getPropertyTypeDef(propertyName);
        final ToBytesSerialiser<Object> serialiser = (null != typeDefinition) ? (ToBytesSerialiser) typeDefinition.getSerialiser() : null;
        final Object property = LengthValueBytesSerialiserUtil.deserialise(serialiser, bytes, delimiter);
        properties.put(propertyName, property);
    }
}

Example 14 with TypeDefinition

use of uk.gov.gchq.gaffer.store.schema.TypeDefinition in project Gaffer by gchq.

the class FilterHandlerTest method shouldFilterBasedOnMatchedVertex.

@Test
public void shouldFilterBasedOnMatchedVertex() throws OperationException {
    // Given
    final Schema schema = new Schema.Builder().edge(TestGroups.EDGE, new SchemaEdgeDefinition.Builder().source("vertex").destination("vertex").property(TestPropertyNames.COUNT, "count.long").build()).type("vertex", new TypeDefinition(String.class)).type("count.long", new TypeDefinition(Long.class)).build();
    given(store.getSchema()).willReturn(schema);
    final Filter filter = new Filter.Builder().input(new Edge.Builder().group(TestGroups.EDGE).source("srcVal1").dest("destVal1").matchedVertex(EdgeId.MatchedVertex.SOURCE).build(), new Edge.Builder().group(TestGroups.EDGE).source("srcVal2").dest("destVal2").matchedVertex(EdgeId.MatchedVertex.SOURCE).build(), new Edge.Builder().group(TestGroups.EDGE).source("srcVal3").dest("destVal3").matchedVertex(EdgeId.MatchedVertex.DESTINATION).build(), new Edge.Builder().group(TestGroups.EDGE).source("srcVal4").dest("destVal4").matchedVertex(EdgeId.MatchedVertex.DESTINATION).build()).edge(TestGroups.EDGE, new ElementFilter.Builder().select(IdentifierType.MATCHED_VERTEX.name()).execute(new IsIn("srcVal1", "destVal3")).build()).build();
    // When
    final Iterable<? extends Element> results = handler.doOperation(filter, context, store);
    // Then
    ElementUtil.assertElementEquals(Arrays.asList(new Edge.Builder().group(TestGroups.EDGE).source("srcVal1").dest("destVal1").matchedVertex(EdgeId.MatchedVertex.SOURCE).build(), new Edge.Builder().group(TestGroups.EDGE).source("srcVal3").dest("destVal3").matchedVertex(EdgeId.MatchedVertex.SOURCE).build()), results);
}

Example 15 with TypeDefinition

use of uk.gov.gchq.gaffer.store.schema.TypeDefinition in project Gaffer by gchq.

the class AbstractCoreKeyAccumuloElementConverter method addDeserialisedProperty.

private int addDeserialisedProperty(final byte[] bytes, final int carriage, final Properties properties, final SchemaElementDefinition elementDefinition, final String propertyName) throws SerialisationException {
    int rtn = carriage;
    final TypeDefinition typeDefinition = elementDefinition.getPropertyTypeDef(propertyName);
    final ToBytesSerialiser serialiser = (null != typeDefinition) ? (ToBytesSerialiser) typeDefinition.getSerialiser() : null;
    if (null != serialiser) {
        final int numBytesForLength = CompactRawSerialisationUtils.decodeVIntSize(bytes[rtn]);
        final int currentPropLength = getCurrentPropLength(bytes, rtn);
        int from = rtn += numBytesForLength;
        rtn += currentPropLength;
        Object deserialisedObject = getDeserialisedObject(serialiser, bytes, from, currentPropLength);
        properties.put(propertyName, deserialisedObject);
    }
    return rtn;
}