Examples with GetRDDOfAllElements uk.gov.gchq.gaffer.spark.operation.scalardd.GetRDDOfAllElements used on opensource projects

Example 6 with GetRDDOfAllElements

use of uk.gov.gchq.gaffer.spark.operation.scalardd.GetRDDOfAllElements in project Gaffer by gchq.

the class FilterToOperationConverterTest method testSpecifyMultiplePropertyFilters.

@Test
public void testSpecifyMultiplePropertyFilters() throws OperationException {
    final Schema schema = getSchema();
    final SQLContext sqlContext = getSqlContext("testSpecifyMultiplePropertyFilters");
    final Filter[] filters = new Filter[2];
    filters[0] = new GreaterThan("property1", 5);
    filters[1] = new LessThan("property4", 8L);
    FiltersToOperationConverter converter = new FiltersToOperationConverter(sqlContext, getViewFromSchema(schema), schema, filters);
    AbstractGetRDD<?> operation = converter.getOperation();
    assertTrue(operation instanceof GetRDDOfAllElements);
    // Only groups ENTITY_GROUP and EDGE_GROUP should be in the view as only they have property1 and property4
    View opView = operation.getView();
    List<ConsumerFunctionContext<String, FilterFunction>> entityPostAggFilters = opView.getEntity(ENTITY_GROUP).getPostAggregationFilterFunctions();
    assertEquals(2, entityPostAggFilters.size());
    final ArrayList<String> expectedProperties = new ArrayList<>();
    expectedProperties.add("property1");
    expectedProperties.add("property4");
    assertEquals(1, entityPostAggFilters.get(0).getSelection().size());
    assertEquals(expectedProperties.get(0), entityPostAggFilters.get(0).getSelection().get(0));
    assertEquals(1, entityPostAggFilters.get(1).getSelection().size());
    assertEquals(expectedProperties.get(1), entityPostAggFilters.get(1).getSelection().get(0));
    final ArrayList<FilterFunction> expectedFunctions = new ArrayList<>();
    expectedFunctions.add(new IsMoreThan(5, false));
    expectedFunctions.add(new IsLessThan(8L, false));
    assertEquals(expectedFunctions.get(0), entityPostAggFilters.get(0).getFunction());
    assertEquals(expectedFunctions.get(1), entityPostAggFilters.get(1).getFunction());
    final List<ConsumerFunctionContext<String, FilterFunction>> edgePostAggFilters = opView.getEdge(EDGE_GROUP).getPostAggregationFilterFunctions();
    assertEquals(2, edgePostAggFilters.size());
    assertEquals(1, edgePostAggFilters.get(0).getSelection().size());
    assertEquals(expectedProperties.get(0), edgePostAggFilters.get(0).getSelection().get(0));
    assertEquals(1, edgePostAggFilters.get(1).getSelection().size());
    assertEquals(expectedProperties.get(1), edgePostAggFilters.get(1).getSelection().get(0));
    sqlContext.sparkContext().stop();
}

Example 7 with GetRDDOfAllElements

use of uk.gov.gchq.gaffer.spark.operation.scalardd.GetRDDOfAllElements in project Gaffer by gchq.

the class FiltersToOperationConverter method applyVertexSourceDestinationFilters.

private AbstractGetRDD<?> applyVertexSourceDestinationFilters(final View view) {
    View clonedView = view.clone();
    AbstractGetRDD<?> operation = null;
    for (final Filter filter : filters) {
        if (filter instanceof EqualTo) {
            final EqualTo equalTo = (EqualTo) filter;
            final String attribute = equalTo.attribute();
            if (attribute.equals(SchemaToStructTypeConverter.VERTEX_COL_NAME)) {
                // Only entities are relevant, so remove any edge groups from the view
                LOGGER.info("Found EqualTo filter with attribute {}, setting views to only contain entity groups", attribute);
                View.Builder viewBuilder = new View.Builder();
                for (final String entityGroup : view.getEntityGroups()) {
                    viewBuilder = viewBuilder.entity(entityGroup);
                }
                clonedView = viewBuilder.build();
                LOGGER.info("Setting operation to GetRDDOfElements");
                operation = new GetRDDOfElements<>(sqlContext.sparkContext(), new EntitySeed(equalTo.value()));
                operation.setView(clonedView);
                break;
            } else if (attribute.equals(SchemaToStructTypeConverter.SRC_COL_NAME) || attribute.equals(SchemaToStructTypeConverter.DST_COL_NAME)) {
                // Only edges are relevant, so remove any entity groups from the view
                LOGGER.info("Found EqualTo filter with attribute {}, setting views to only contain edge groups", attribute);
                View.Builder viewBuilder = new View.Builder();
                for (final String edgeGroup : view.getEdgeGroups()) {
                    viewBuilder = viewBuilder.edge(edgeGroup);
                }
                clonedView = viewBuilder.build();
                LOGGER.info("Setting operation to GetRDDOfElements");
                operation = new GetRDDOfElements<>(sqlContext.sparkContext(), new EntitySeed(equalTo.value()));
                operation.setView(clonedView);
                break;
            }
        }
    }
    if (operation == null) {
        LOGGER.debug("Setting operation to GetRDDOfAllElements");
        operation = new GetRDDOfAllElements(sqlContext.sparkContext());
        operation.setView(clonedView);
    }
    return operation;
}

Example 8 with GetRDDOfAllElements

use of uk.gov.gchq.gaffer.spark.operation.scalardd.GetRDDOfAllElements in project Gaffer by gchq.

the class AccumuloStoreRelation method buildScan.

/**
     * Creates a <code>DataFrame</code> of all {@link Element}s from the specified groups.
     *
     * @return An {@link RDD} of {@link Row}s containing {@link Element}s whose group is in <code>groups</code>.
     */
@Override
public RDD<Row> buildScan() {
    try {
        LOGGER.info("Building GetRDDOfAllElements with view set to groups {}", StringUtils.join(groups, ','));
        final GetRDDOfAllElements operation = new GetRDDOfAllElements(sqlContext.sparkContext());
        operation.setView(view);
        final RDD<Element> rdd = store.execute(operation, user);
        return rdd.map(new ConvertElementToRow(usedProperties, propertyNeedsConversion, converterByProperty), ClassTagConstants.ROW_CLASS_TAG);
    } catch (final OperationException e) {
        LOGGER.error("OperationException while executing operation: {}", e);
        return null;
    }
}

Example 9 with GetRDDOfAllElements

use of uk.gov.gchq.gaffer.spark.operation.scalardd.GetRDDOfAllElements in project Gaffer by gchq.

the class FilterToOperationConverterTest method testTwoGroups.

@Test
public void testTwoGroups() throws OperationException {
    final Schema schema = getSchema();
    final SQLContext sqlContext = getSqlContext("testTwoGroups");
    final Filter[] filters = new Filter[1];
    final Filter left = new EqualTo(SchemaToStructTypeConverter.GROUP, ENTITY_GROUP);
    final Filter right = new EqualTo(SchemaToStructTypeConverter.GROUP, EDGE_GROUP2);
    filters[0] = new Or(left, right);
    final FiltersToOperationConverter converter = new FiltersToOperationConverter(sqlContext, getViewFromSchema(schema), schema, filters);
    final AbstractGetRDD<?> operation = converter.getOperation();
    assertTrue(operation instanceof GetRDDOfAllElements);
    assertEquals(Collections.singleton(ENTITY_GROUP), operation.getView().getEntityGroups());
    assertEquals(Collections.singleton(EDGE_GROUP2), operation.getView().getEdgeGroups());
    sqlContext.sparkContext().stop();
}