Search in sources :

Example 1 with TupleAdaptedPredicate

use of uk.gov.gchq.koryphe.tuple.predicate.TupleAdaptedPredicate in project Gaffer by gchq.

the class ElementFilterTest method shouldBuildFilter.

@Test
public void shouldBuildFilter() {
    // Given
    final String property1 = "property 1";
    final String property2a = "property 2a";
    final String property2b = "property 2b";
    final String property3 = "property 3";
    final Predicate func1 = mock(Predicate.class);
    final Predicate func2 = mock(Predicate.class);
    final Predicate func3 = mock(Predicate.class);
    // When - check you can build the selection/function in any order,
    // although normally it will be done - select then execute.
    final ElementFilter filter = new ElementFilter.Builder().select(property1).execute(func1).select(property2a, property2b).execute(func2).select(property3).execute(func3).build();
    // Then
    int i = 0;
    TupleAdaptedPredicate<String, ?> adaptedFunction = filter.getComponents().get(i++);
    assertThat(adaptedFunction.getSelection()).hasSize(1);
    assertEquals(property1, adaptedFunction.getSelection()[0]);
    assertSame(func1, adaptedFunction.getPredicate());
    adaptedFunction = filter.getComponents().get(i++);
    assertThat(adaptedFunction.getSelection()).hasSize(2);
    assertEquals(property2a, adaptedFunction.getSelection()[0]);
    assertEquals(property2b, adaptedFunction.getSelection()[1]);
    assertSame(func2, adaptedFunction.getPredicate());
    adaptedFunction = filter.getComponents().get(i++);
    assertSame(func3, adaptedFunction.getPredicate());
    assertThat(adaptedFunction.getSelection()).hasSize(1);
    assertEquals(property3, adaptedFunction.getSelection()[0]);
    assertEquals(i, filter.getComponents().size());
}
Also used : Predicate(java.util.function.Predicate) TupleAdaptedPredicate(uk.gov.gchq.koryphe.tuple.predicate.TupleAdaptedPredicate) JSONSerialisationTest(uk.gov.gchq.gaffer.JSONSerialisationTest) Test(org.junit.jupiter.api.Test)

Example 2 with TupleAdaptedPredicate

use of uk.gov.gchq.koryphe.tuple.predicate.TupleAdaptedPredicate in project Gaffer by gchq.

the class FilterToOperationConverterTest method testSpecifyVertexAndPropertyFilter.

@Test
public void testSpecifyVertexAndPropertyFilter() {
    final Schema schema = getSchema();
    final SparkSession sparkSession = SparkSessionProvider.getSparkSession();
    // Specify vertex and a filter on property1
    Filter[] filters = new Filter[2];
    filters[0] = new GreaterThan("property1", 5);
    filters[1] = new EqualTo(SchemaToStructTypeConverter.VERTEX_COL_NAME, "0");
    FiltersToOperationConverter converter = new FiltersToOperationConverter(getViewFromSchema(schema), schema, filters);
    Operation operation = converter.getOperation();
    assertTrue(operation instanceof GetRDDOfElements);
    assertEquals(1, ((GraphFilters) operation).getView().getEntityGroups().size());
    assertEquals(0, ((GraphFilters) operation).getView().getEdgeGroups().size());
    final Set<EntityId> seeds = new HashSet<>();
    for (final Object seed : ((GetRDDOfElements) operation).getInput()) {
        seeds.add((EntitySeed) seed);
    }
    assertEquals(Collections.singleton(new EntitySeed("0")), seeds);
    View opView = ((GraphFilters) operation).getView();
    List<TupleAdaptedPredicate<String, ?>> entityPostAggFilters = opView.getEntity(ENTITY_GROUP).getPostAggregationFilterFunctions();
    assertThat(entityPostAggFilters).hasSize(1);
    final ArrayList<String> expectedProperties = new ArrayList<>();
    expectedProperties.add("property1");
    assertThat(entityPostAggFilters.get(0).getSelection()).hasSize(1);
    assertEquals(expectedProperties.get(0), entityPostAggFilters.get(0).getSelection()[0]);
    final ArrayList<Predicate> expectedFunctions = new ArrayList<>();
    expectedFunctions.add(new IsMoreThan(5, false));
    assertEquals(expectedFunctions.get(0), entityPostAggFilters.get(0).getPredicate());
    // Specify vertex and filters on properties property1 and property4
    filters = new Filter[3];
    filters[0] = new GreaterThan("property1", 5);
    filters[1] = new EqualTo(SchemaToStructTypeConverter.VERTEX_COL_NAME, "0");
    filters[2] = new LessThan("property4", 8);
    converter = new FiltersToOperationConverter(getViewFromSchema(schema), schema, filters);
    operation = converter.getOperation();
    assertTrue(operation instanceof GetRDDOfElements);
    assertEquals(1, ((GraphFilters) operation).getView().getEntityGroups().size());
    assertEquals(0, ((GraphFilters) operation).getView().getEdgeGroups().size());
    seeds.clear();
    for (final Object seed : ((GetRDDOfElements) operation).getInput()) {
        seeds.add((EntitySeed) seed);
    }
    assertEquals(Collections.singleton(new EntitySeed("0")), seeds);
    opView = ((GraphFilters) operation).getView();
    entityPostAggFilters = opView.getEntity(ENTITY_GROUP).getPostAggregationFilterFunctions();
    assertThat(entityPostAggFilters).hasSize(2);
    expectedProperties.clear();
    expectedProperties.add("property1");
    expectedProperties.add("property4");
    assertThat(entityPostAggFilters.get(0).getSelection()).hasSize(1);
    assertEquals(expectedProperties.get(0), entityPostAggFilters.get(0).getSelection()[0]);
    assertThat(entityPostAggFilters.get(1).getSelection()).hasSize(1);
    assertEquals(expectedProperties.get(1), entityPostAggFilters.get(1).getSelection()[0]);
    expectedFunctions.clear();
    expectedFunctions.add(new IsMoreThan(5, false));
    expectedFunctions.add(new IsLessThan(8, false));
    assertEquals(expectedFunctions.get(0), entityPostAggFilters.get(0).getPredicate());
    assertEquals(expectedFunctions.get(1), entityPostAggFilters.get(1).getPredicate());
}
Also used : SparkSession(org.apache.spark.sql.SparkSession) Schema(uk.gov.gchq.gaffer.store.schema.Schema) ArrayList(java.util.ArrayList) GraphFilters(uk.gov.gchq.gaffer.operation.graph.GraphFilters) Operation(uk.gov.gchq.gaffer.operation.Operation) TupleAdaptedPredicate(uk.gov.gchq.koryphe.tuple.predicate.TupleAdaptedPredicate) Predicate(java.util.function.Predicate) IsLessThan(uk.gov.gchq.koryphe.impl.predicate.IsLessThan) LessThan(org.apache.spark.sql.sources.LessThan) IsLessThan(uk.gov.gchq.koryphe.impl.predicate.IsLessThan) GreaterThan(org.apache.spark.sql.sources.GreaterThan) GetRDDOfElements(uk.gov.gchq.gaffer.spark.operation.scalardd.GetRDDOfElements) HashSet(java.util.HashSet) TupleAdaptedPredicate(uk.gov.gchq.koryphe.tuple.predicate.TupleAdaptedPredicate) View(uk.gov.gchq.gaffer.data.elementdefinition.view.View) EqualTo(org.apache.spark.sql.sources.EqualTo) EntityId(uk.gov.gchq.gaffer.data.element.id.EntityId) Filter(org.apache.spark.sql.sources.Filter) EntitySeed(uk.gov.gchq.gaffer.operation.data.EntitySeed) IsMoreThan(uk.gov.gchq.koryphe.impl.predicate.IsMoreThan) Test(org.junit.jupiter.api.Test)

Example 3 with TupleAdaptedPredicate

use of uk.gov.gchq.koryphe.tuple.predicate.TupleAdaptedPredicate in project Gaffer by gchq.

the class FilterToOperationConverterTest method testSpecifyPropertyFilters.

@Test
public void testSpecifyPropertyFilters() {
    final Schema schema = getSchema();
    final SparkSession sparkSession = SparkSessionProvider.getSparkSession();
    final Filter[] filters = new Filter[1];
    // GreaterThan
    filters[0] = new GreaterThan("property1", 5);
    FiltersToOperationConverter converter = new FiltersToOperationConverter(getViewFromSchema(schema), schema, filters);
    Operation operation = converter.getOperation();
    assertTrue(operation instanceof GetRDDOfAllElements);
    View opView = ((GraphFilters) operation).getView();
    List<TupleAdaptedPredicate<String, ?>> entityPostAggFilters = opView.getEntity(ENTITY_GROUP).getPostAggregationFilterFunctions();
    assertThat(entityPostAggFilters).hasSize(1);
    assertArrayEquals(new String[] { "property1" }, entityPostAggFilters.get(0).getSelection());
    assertEquals(new IsMoreThan(5, false), entityPostAggFilters.get(0).getPredicate());
    for (final String edgeGroup : EDGE_GROUPS) {
        final List<TupleAdaptedPredicate<String, ?>> edgePostAggFilters = opView.getEdge(edgeGroup).getPostAggregationFilterFunctions();
        assertThat(edgePostAggFilters).hasSize(1);
        assertArrayEquals(new String[] { "property1" }, edgePostAggFilters.get(0).getSelection());
        assertEquals(new IsMoreThan(5, false), edgePostAggFilters.get(0).getPredicate());
    }
    // LessThan
    filters[0] = new LessThan("property4", 8L);
    converter = new FiltersToOperationConverter(getViewFromSchema(schema), schema, filters);
    operation = converter.getOperation();
    assertTrue(operation instanceof GetRDDOfAllElements);
    // Only groups ENTITY_GROUP and EDGE_GROUP should be in the view as only they have property4
    opView = ((GraphFilters) operation).getView();
    entityPostAggFilters = opView.getEntity(ENTITY_GROUP).getPostAggregationFilterFunctions();
    assertThat(entityPostAggFilters).hasSize(1);
    assertArrayEquals(new String[] { "property4" }, entityPostAggFilters.get(0).getSelection());
    assertEquals(new IsLessThan(8L, false), entityPostAggFilters.get(0).getPredicate());
    List<TupleAdaptedPredicate<String, ?>> edgePostAggFilters = opView.getEdge(EDGE_GROUP).getPostAggregationFilterFunctions();
    assertThat(edgePostAggFilters).hasSize(1);
    assertArrayEquals(new String[] { "property4" }, edgePostAggFilters.get(0).getSelection());
    assertEquals(new IsLessThan(8L, false), edgePostAggFilters.get(0).getPredicate());
    // And
    final Filter left = new GreaterThan("property1", 5);
    final Filter right = new GreaterThan("property4", 8L);
    filters[0] = new And(left, right);
    converter = new FiltersToOperationConverter(getViewFromSchema(schema), schema, filters);
    operation = converter.getOperation();
    assertTrue(operation instanceof GetRDDOfAllElements);
    // Only groups ENTITY_GROUP and EDGE_GROUP should be in the view as only they have property1 and property4
    opView = ((GraphFilters) operation).getView();
    entityPostAggFilters = opView.getEntity(ENTITY_GROUP).getPostAggregationFilterFunctions();
    assertThat(entityPostAggFilters).hasSize(2);
    final ArrayList<String> expectedProperties = new ArrayList<>();
    expectedProperties.add("property1");
    expectedProperties.add("property4");
    assertThat(entityPostAggFilters.get(0).getSelection()).hasSize(1);
    assertEquals(expectedProperties.get(0), entityPostAggFilters.get(0).getSelection()[0]);
    assertThat(entityPostAggFilters.get(1).getSelection()).hasSize(1);
    assertEquals(expectedProperties.get(1), entityPostAggFilters.get(1).getSelection()[0]);
    final ArrayList<Predicate> expectedFunctions = new ArrayList<>();
    expectedFunctions.add(new IsMoreThan(5, false));
    expectedFunctions.add(new IsMoreThan(8L, false));
    assertEquals(expectedFunctions.get(0), entityPostAggFilters.get(0).getPredicate());
    assertEquals(expectedFunctions.get(1), entityPostAggFilters.get(1).getPredicate());
    edgePostAggFilters = opView.getEdge(EDGE_GROUP).getPostAggregationFilterFunctions();
    assertThat(edgePostAggFilters).hasSize(2);
    assertThat(edgePostAggFilters.get(0).getSelection()).hasSize(1);
    assertEquals(expectedProperties.get(0), edgePostAggFilters.get(0).getSelection()[0]);
    assertThat(edgePostAggFilters.get(1).getSelection()).hasSize(1);
    assertEquals(expectedProperties.get(1), edgePostAggFilters.get(1).getSelection()[0]);
}
Also used : TupleAdaptedPredicate(uk.gov.gchq.koryphe.tuple.predicate.TupleAdaptedPredicate) SparkSession(org.apache.spark.sql.SparkSession) Schema(uk.gov.gchq.gaffer.store.schema.Schema) ArrayList(java.util.ArrayList) GraphFilters(uk.gov.gchq.gaffer.operation.graph.GraphFilters) Operation(uk.gov.gchq.gaffer.operation.Operation) View(uk.gov.gchq.gaffer.data.elementdefinition.view.View) TupleAdaptedPredicate(uk.gov.gchq.koryphe.tuple.predicate.TupleAdaptedPredicate) Predicate(java.util.function.Predicate) IsLessThan(uk.gov.gchq.koryphe.impl.predicate.IsLessThan) LessThan(org.apache.spark.sql.sources.LessThan) IsLessThan(uk.gov.gchq.koryphe.impl.predicate.IsLessThan) Filter(org.apache.spark.sql.sources.Filter) GreaterThan(org.apache.spark.sql.sources.GreaterThan) GetRDDOfAllElements(uk.gov.gchq.gaffer.spark.operation.scalardd.GetRDDOfAllElements) And(org.apache.spark.sql.sources.And) IsMoreThan(uk.gov.gchq.koryphe.impl.predicate.IsMoreThan) Test(org.junit.jupiter.api.Test)

Example 4 with TupleAdaptedPredicate

use of uk.gov.gchq.koryphe.tuple.predicate.TupleAdaptedPredicate in project Gaffer by gchq.

the class FilterToOperationConverterTest method testSpecifySourceOrDestinationAndPropertyFilter.

@Test
public void testSpecifySourceOrDestinationAndPropertyFilter() {
    final Schema schema = getSchema();
    final SparkSession sparkSession = SparkSessionProvider.getSparkSession();
    // Specify src and a filter on property1
    Filter[] filters = new Filter[2];
    filters[0] = new GreaterThan("property1", 5);
    filters[1] = new EqualTo(SchemaToStructTypeConverter.SRC_COL_NAME, "0");
    FiltersToOperationConverter converter = new FiltersToOperationConverter(getViewFromSchema(schema), schema, filters);
    Operation operation = converter.getOperation();
    assertTrue(operation instanceof GetRDDOfElements);
    assertEquals(0, ((GraphFilters) operation).getView().getEntityGroups().size());
    assertEquals(2, ((GraphFilters) operation).getView().getEdgeGroups().size());
    final Set<EntityId> seeds = new HashSet<>();
    for (final Object seed : ((GetRDDOfElements) operation).getInput()) {
        seeds.add((EntitySeed) seed);
    }
    assertEquals(Collections.singleton(new EntitySeed("0")), seeds);
    View opView = ((GraphFilters) operation).getView();
    for (final String edgeGroup : EDGE_GROUPS) {
        final List<TupleAdaptedPredicate<String, ?>> edgePostAggFilters = opView.getEdge(edgeGroup).getPostAggregationFilterFunctions();
        assertThat(edgePostAggFilters).hasSize(1);
        assertArrayEquals(new String[] { "property1" }, edgePostAggFilters.get(0).getSelection());
        assertEquals(new IsMoreThan(5, false), edgePostAggFilters.get(0).getPredicate());
    }
    // Specify src and filters on property1 and property4
    filters = new Filter[3];
    filters[0] = new GreaterThan("property1", 5);
    filters[1] = new EqualTo(SchemaToStructTypeConverter.SRC_COL_NAME, "0");
    filters[2] = new LessThan("property4", 8);
    converter = new FiltersToOperationConverter(getViewFromSchema(schema), schema, filters);
    operation = converter.getOperation();
    assertTrue(operation instanceof GetRDDOfElements);
    assertEquals(0, ((GraphFilters) operation).getView().getEntityGroups().size());
    assertEquals(1, ((GraphFilters) operation).getView().getEdgeGroups().size());
    seeds.clear();
    for (final Object seed : ((GetRDDOfElements) operation).getInput()) {
        seeds.add((EntitySeed) seed);
    }
    assertEquals(Collections.singleton(new EntitySeed("0")), seeds);
    opView = ((GraphFilters) operation).getView();
    final List<TupleAdaptedPredicate<String, ?>> entityPostAggFilters = opView.getEdge(EDGE_GROUP).getPostAggregationFilterFunctions();
    assertThat(entityPostAggFilters).hasSize(2);
    final List<String> expectedProperties = new ArrayList<>();
    expectedProperties.add("property1");
    expectedProperties.add("property4");
    assertThat(entityPostAggFilters.get(0).getSelection()).hasSize(1);
    assertEquals(expectedProperties.get(0), entityPostAggFilters.get(0).getSelection()[0]);
    assertEquals(new IsMoreThan(5, false), entityPostAggFilters.get(0).getPredicate());
    assertThat(entityPostAggFilters.get(1).getSelection()).hasSize(1);
    assertEquals(expectedProperties.get(1), entityPostAggFilters.get(1).getSelection()[0]);
    assertEquals(new IsLessThan(8, false), entityPostAggFilters.get(1).getPredicate());
}
Also used : TupleAdaptedPredicate(uk.gov.gchq.koryphe.tuple.predicate.TupleAdaptedPredicate) SparkSession(org.apache.spark.sql.SparkSession) Schema(uk.gov.gchq.gaffer.store.schema.Schema) ArrayList(java.util.ArrayList) GraphFilters(uk.gov.gchq.gaffer.operation.graph.GraphFilters) Operation(uk.gov.gchq.gaffer.operation.Operation) View(uk.gov.gchq.gaffer.data.elementdefinition.view.View) EqualTo(org.apache.spark.sql.sources.EqualTo) EntityId(uk.gov.gchq.gaffer.data.element.id.EntityId) IsLessThan(uk.gov.gchq.koryphe.impl.predicate.IsLessThan) LessThan(org.apache.spark.sql.sources.LessThan) IsLessThan(uk.gov.gchq.koryphe.impl.predicate.IsLessThan) Filter(org.apache.spark.sql.sources.Filter) GreaterThan(org.apache.spark.sql.sources.GreaterThan) EntitySeed(uk.gov.gchq.gaffer.operation.data.EntitySeed) IsMoreThan(uk.gov.gchq.koryphe.impl.predicate.IsMoreThan) GetRDDOfElements(uk.gov.gchq.gaffer.spark.operation.scalardd.GetRDDOfElements) HashSet(java.util.HashSet) Test(org.junit.jupiter.api.Test)

Example 5 with TupleAdaptedPredicate

use of uk.gov.gchq.koryphe.tuple.predicate.TupleAdaptedPredicate in project Gaffer by gchq.

the class FilterToOperationConverterTest method testSpecifyMultiplePropertyFilters.

@Test
public void testSpecifyMultiplePropertyFilters() {
    final Schema schema = getSchema();
    final SparkSession sparkSession = SparkSessionProvider.getSparkSession();
    final Filter[] filters = new Filter[2];
    filters[0] = new GreaterThan("property1", 5);
    filters[1] = new LessThan("property4", 8L);
    FiltersToOperationConverter converter = new FiltersToOperationConverter(getViewFromSchema(schema), schema, filters);
    Operation operation = converter.getOperation();
    assertTrue(operation instanceof GetRDDOfAllElements);
    // Only groups ENTITY_GROUP and EDGE_GROUP should be in the view as only they have property1 and property4
    View opView = ((GraphFilters) operation).getView();
    List<TupleAdaptedPredicate<String, ?>> entityPostAggFilters = opView.getEntity(ENTITY_GROUP).getPostAggregationFilterFunctions();
    assertThat(entityPostAggFilters).hasSize(2);
    final ArrayList<String> expectedProperties = new ArrayList<>();
    expectedProperties.add("property1");
    expectedProperties.add("property4");
    assertThat(entityPostAggFilters.get(0).getSelection()).hasSize(1);
    assertEquals(expectedProperties.get(0), entityPostAggFilters.get(0).getSelection()[0]);
    assertThat(entityPostAggFilters.get(1).getSelection()).hasSize(1);
    assertEquals(expectedProperties.get(1), entityPostAggFilters.get(1).getSelection()[0]);
    final ArrayList<Predicate> expectedFunctions = new ArrayList<>();
    expectedFunctions.add(new IsMoreThan(5, false));
    expectedFunctions.add(new IsLessThan(8L, false));
    assertEquals(expectedFunctions.get(0), entityPostAggFilters.get(0).getPredicate());
    assertEquals(expectedFunctions.get(1), entityPostAggFilters.get(1).getPredicate());
    final List<TupleAdaptedPredicate<String, ?>> edgePostAggFilters = opView.getEdge(EDGE_GROUP).getPostAggregationFilterFunctions();
    assertThat(edgePostAggFilters).hasSize(2);
    assertThat(edgePostAggFilters.get(0).getSelection()).hasSize(1);
    assertEquals(expectedProperties.get(0), edgePostAggFilters.get(0).getSelection()[0]);
    assertThat(edgePostAggFilters.get(1).getSelection()).hasSize(1);
    assertEquals(expectedProperties.get(1), edgePostAggFilters.get(1).getSelection()[0]);
}
Also used : TupleAdaptedPredicate(uk.gov.gchq.koryphe.tuple.predicate.TupleAdaptedPredicate) SparkSession(org.apache.spark.sql.SparkSession) Schema(uk.gov.gchq.gaffer.store.schema.Schema) ArrayList(java.util.ArrayList) GraphFilters(uk.gov.gchq.gaffer.operation.graph.GraphFilters) Operation(uk.gov.gchq.gaffer.operation.Operation) View(uk.gov.gchq.gaffer.data.elementdefinition.view.View) TupleAdaptedPredicate(uk.gov.gchq.koryphe.tuple.predicate.TupleAdaptedPredicate) Predicate(java.util.function.Predicate) IsLessThan(uk.gov.gchq.koryphe.impl.predicate.IsLessThan) LessThan(org.apache.spark.sql.sources.LessThan) IsLessThan(uk.gov.gchq.koryphe.impl.predicate.IsLessThan) Filter(org.apache.spark.sql.sources.Filter) GreaterThan(org.apache.spark.sql.sources.GreaterThan) GetRDDOfAllElements(uk.gov.gchq.gaffer.spark.operation.scalardd.GetRDDOfAllElements) IsMoreThan(uk.gov.gchq.koryphe.impl.predicate.IsMoreThan) Test(org.junit.jupiter.api.Test)

Aggregations

TupleAdaptedPredicate (uk.gov.gchq.koryphe.tuple.predicate.TupleAdaptedPredicate)13 Test (org.junit.jupiter.api.Test)8 Predicate (java.util.function.Predicate)7 IsMoreThan (uk.gov.gchq.koryphe.impl.predicate.IsMoreThan)7 ArrayList (java.util.ArrayList)6 IsLessThan (uk.gov.gchq.koryphe.impl.predicate.IsLessThan)6 GreaterThan (org.apache.spark.sql.sources.GreaterThan)5 LessThan (org.apache.spark.sql.sources.LessThan)5 View (uk.gov.gchq.gaffer.data.elementdefinition.view.View)5 SparkSession (org.apache.spark.sql.SparkSession)4 Filter (org.apache.spark.sql.sources.Filter)4 Operation (uk.gov.gchq.gaffer.operation.Operation)4 GraphFilters (uk.gov.gchq.gaffer.operation.graph.GraphFilters)4 Schema (uk.gov.gchq.gaffer.store.schema.Schema)4 HashSet (java.util.HashSet)3 FilterPredicate (org.apache.parquet.filter2.predicate.FilterPredicate)3 EqualTo (org.apache.spark.sql.sources.EqualTo)3 And (org.apache.spark.sql.sources.And)2 ElementFilter (uk.gov.gchq.gaffer.data.element.function.ElementFilter)2 EntityId (uk.gov.gchq.gaffer.data.element.id.EntityId)2