Examples with SchemaEdgeDefinition uk.gov.gchq.gaffer.store.schema.SchemaEdgeDefinition used on opensource projects

Example 1 with SchemaEdgeDefinition

use of uk.gov.gchq.gaffer.store.schema.SchemaEdgeDefinition in project Gaffer by gchq.

the class ByteEntityBloomElementFunctorTest method setup.

@BeforeEach
public void setup() {
    schema = new Schema.Builder().vertexSerialiser(new JavaSerialiser()).edge(TestGroups.EDGE, new SchemaEdgeDefinition()).entity(TestGroups.ENTITY, new SchemaEntityDefinition()).build();
    elementConverter = new ByteEntityAccumuloElementConverter(schema);
}

Example 2 with SchemaEdgeDefinition

use of uk.gov.gchq.gaffer.store.schema.SchemaEdgeDefinition in project Gaffer by gchq.

the class Gaffer1BloomElementFunctorTest method setup.

@BeforeEach
public void setup() {
    schema = new Schema.Builder().vertexSerialiser(new JavaSerialiser()).edge(TestGroups.EDGE, new SchemaEdgeDefinition()).entity(TestGroups.ENTITY, new SchemaEntityDefinition()).build();
    elementConverter = new ClassicAccumuloElementConverter(schema);
}

Example 3 with SchemaEdgeDefinition

use of uk.gov.gchq.gaffer.store.schema.SchemaEdgeDefinition in project Gaffer by gchq.

the class EdgeSerialiserTest method setUp.

@BeforeAll
public static void setUp() {
    final SchemaEdgeDefinition edgeDef = new SchemaEdgeDefinition.Builder().build();
    schema = new Schema.Builder().vertexSerialiser(new StringSerialiser()).edge(TestGroups.EDGE, edgeDef).build();
    serialiser = new EdgeSerialiser(schema);
}

Example 4 with SchemaEdgeDefinition

use of uk.gov.gchq.gaffer.store.schema.SchemaEdgeDefinition in project Gaffer by gchq.

the class SchemaToStructTypeConverter method buildSchema.

private void buildSchema() {
    LOGGER.info("Building Spark SQL schema for groups {}", StringUtils.join(groups, ','));
    for (final String group : groups) {
        final SchemaElementDefinition elementDefn = schema.getElement(group);
        final List<StructField> structFieldList = new ArrayList<>();
        if (elementDefn instanceof SchemaEntityDefinition) {
            entityOrEdgeByGroup.put(group, EntityOrEdge.ENTITY);
            final SchemaEntityDefinition entityDefinition = (SchemaEntityDefinition) elementDefn;
            final String vertexClass = schema.getType(entityDefinition.getVertex()).getFullClassString();
            final DataType vertexType = getType(vertexClass);
            if (null == vertexType) {
                throw new RuntimeException("Vertex must be a recognised type: found " + vertexClass);
            }
            LOGGER.info("Group {} is an entity group - {} is of type {}", group, VERTEX_COL_NAME, vertexType);
            structFieldList.add(new StructField(VERTEX_COL_NAME, vertexType, true, Metadata.empty()));
        } else {
            entityOrEdgeByGroup.put(group, EntityOrEdge.EDGE);
            final SchemaEdgeDefinition edgeDefinition = (SchemaEdgeDefinition) elementDefn;
            final String srcClass = schema.getType(edgeDefinition.getSource()).getFullClassString();
            final String dstClass = schema.getType(edgeDefinition.getDestination()).getFullClassString();
            final DataType srcType = getType(srcClass);
            final DataType dstType = getType(dstClass);
            if (null == srcType || null == dstType) {
                throw new RuntimeException("Both source and destination must be recognised types: source was " + srcClass + " destination was " + dstClass);
            }
            LOGGER.info("Group {} is an edge group - {} is of type {}, {} is of type {}", group, SRC_COL_NAME, srcType, DST_COL_NAME, dstType);
            structFieldList.add(new StructField(SRC_COL_NAME, srcType, true, Metadata.empty()));
            structFieldList.add(new StructField(DST_COL_NAME, dstType, true, Metadata.empty()));
            structFieldList.add(new StructField(DIRECTED_COL_NAME, DataTypes.BooleanType, true, Metadata.empty()));
            structFieldList.add(new StructField(MATCHED_VERTEX_COL_NAME, DataTypes.StringType, true, Metadata.empty()));
        }
        final Set<String> properties = elementDefn.getProperties();
        for (final String property : properties) {
            if (!ReservedPropertyNames.contains(property)) {
                // Check if property is of a known type that can be handled by default
                final String propertyClass = elementDefn.getPropertyClass(property).getCanonicalName();
                DataType propertyType = getType(propertyClass);
                if (null != propertyType) {
                    propertyNeedsConversion.put(property, needsConversion(propertyClass));
                    structFieldList.add(new StructField(property, propertyType, true, Metadata.empty()));
                    LOGGER.info("Property {} is of type {}", property, propertyType);
                } else {
                    // Check if any of the provided converters can handle it
                    if (null != converters) {
                        for (final Converter converter : converters) {
                            if (converter.canHandle(elementDefn.getPropertyClass(property))) {
                                propertyNeedsConversion.put(property, true);
                                propertyType = converter.convertedType();
                                converterByProperty.put(property, converter);
                                structFieldList.add(new StructField(property, propertyType, true, Metadata.empty()));
                                LOGGER.info("Property {} of type {} will be converted by {} to {}", property, propertyClass, converter.getClass().getName(), propertyType);
                                break;
                            }
                        }
                        if (null == propertyType) {
                            LOGGER.warn("Ignoring property {} as it is not a recognised type and none of the provided " + "converters can handle it", property);
                        }
                    }
                }
            } else {
                LOGGER.warn("Ignoring property {} in group {} as it shares a name with a reserved keyword used to define " + "elements.", property, group);
            }
        }
        structTypeByGroup.put(group, new StructType(structFieldList.toArray(new StructField[structFieldList.size()])));
    }
    // Create reverse map of field name to StructField
    final Map<String, Set<StructField>> fieldToStructs = new HashMap<>();
    for (final String group : groups) {
        final StructType groupSchema = structTypeByGroup.get(group);
        for (final String field : groupSchema.fieldNames()) {
            fieldToStructs.computeIfAbsent(field, k -> new HashSet<StructField>());
            fieldToStructs.get(field).add(groupSchema.apply(field));
        }
    }
    // Check consistency, i.e. if the same field appears in multiple groups then the types are consistent
    for (final Entry<String, Set<StructField>> entry : fieldToStructs.entrySet()) {
        final Set<StructField> schemas = entry.getValue();
        if (schemas.size() > 1) {
            throw new IllegalArgumentException("Inconsistent fields: the field " + entry.getKey() + " has more than one definition: " + StringUtils.join(schemas, ','));
        }
    }
    // Merge schemas for groups together - fields should appear in the order the groups were provided
    final LinkedHashSet<StructField> fields = new LinkedHashSet<>();
    fields.add(new StructField(GROUP, DataTypes.StringType, false, Metadata.empty()));
    usedProperties.add(GROUP);
    for (final String group : groups) {
        final StructType groupSchema = structTypeByGroup.get(group);
        for (final String field : groupSchema.fieldNames()) {
            final StructField struct = groupSchema.apply(field);
            // Add struct to fields unless it has already been added
            if (!fields.contains(struct)) {
                fields.add(struct);
                usedProperties.add(field);
            }
        }
    }
    structType = new StructType(fields.toArray(new StructField[fields.size()]));
    LOGGER.info("Schema is {}", structType);
    LOGGER.debug("properties -> conversion: {}", StringUtils.join(propertyNeedsConversion.entrySet(), ','));
}

Example 5 with SchemaEdgeDefinition

use of uk.gov.gchq.gaffer.store.schema.SchemaEdgeDefinition in project Gaffer by gchq.

the class AbstractCoreKeyIteratorSettingsFactory method queryTimeAggregatorRequired.

public boolean queryTimeAggregatorRequired(final View view, final AccumuloStore store) {
    Schema schema = store.getSchema();
    if (!schema.isAggregationEnabled()) {
        return false;
    }
    String visibilityProp = schema.getVisibilityProperty();
    for (final String edgeGroup : view.getEdgeGroups()) {
        SchemaEdgeDefinition edgeDefinition = schema.getEdge(edgeGroup);
        if (null != edgeDefinition) {
            if (edgeDefinition.containsProperty(visibilityProp)) {
                return true;
            }
            ViewElementDefinition viewElementDefinition = view.getEdge(edgeGroup);
            if ((null != viewElementDefinition && null != viewElementDefinition.getGroupBy()) && (edgeDefinition.getGroupBy().size() != viewElementDefinition.getGroupBy().size())) {
                return true;
            }
        }
    }
    for (final String entityGroup : view.getEntityGroups()) {
        SchemaEntityDefinition entityDefinition = schema.getEntity(entityGroup);
        if (null != entityDefinition) {
            if (entityDefinition.containsProperty(visibilityProp)) {
                return true;
            }
            ViewElementDefinition viewElementDefinition = view.getElement(entityGroup);
            if ((null != viewElementDefinition.getGroupBy()) && (entityDefinition.getGroupBy().size() != viewElementDefinition.getGroupBy().size())) {
                return true;
            }
        }
    }
    return false;
}