Examples with FilterPredicate org.apache.parquet.filter2.predicate.FilterPredicate used on opensource projects

Example 26 with FilterPredicate

use of org.apache.parquet.filter2.predicate.FilterPredicate in project parquet-mr by apache.

the class DictionaryFilterTest method testAnd.

@Test
public void testAnd() throws Exception {
    BinaryColumn col = binaryColumn("binary_field");
    // both evaluate to false (no upper-case letters are in the dictionary)
    FilterPredicate B = eq(col, Binary.fromString("B"));
    FilterPredicate C = eq(col, Binary.fromString("C"));
    // both evaluate to true (all lower-case letters are in the dictionary)
    FilterPredicate x = eq(col, Binary.fromString("x"));
    FilterPredicate y = eq(col, Binary.fromString("y"));
    assertTrue("Should drop when either predicate must be false", canDrop(and(B, y), ccmd, dictionaries));
    assertTrue("Should drop when either predicate must be false", canDrop(and(x, C), ccmd, dictionaries));
    assertTrue("Should drop when either predicate must be false", canDrop(and(B, C), ccmd, dictionaries));
    assertFalse("Should not drop when either predicate could be true", canDrop(and(x, y), ccmd, dictionaries));
}

Example 27 with FilterPredicate

use of org.apache.parquet.filter2.predicate.FilterPredicate in project presto by prestodb.

the class TupleDomainParquetPredicate method getParquetUserDefinedPredicate.

public FilterPredicate getParquetUserDefinedPredicate() {
    FilterPredicate filter = null;
    // we assume the relation cross domains are 'or'
    for (RichColumnDescriptor column : columns) {
        Domain domain = effectivePredicate.getDomains().get().get(column);
        if (domain == null || domain.isNone()) {
            continue;
        }
        if (domain.isAll()) {
            continue;
        }
        FilterPredicate columnFilter = FilterApi.userDefined(FilterApi.intColumn(ColumnPath.get(column.getPath()).toDotString()), new ParquetUserDefinedPredicateTupleDomain(domain));
        if (filter == null) {
            filter = columnFilter;
        } else {
            filter = FilterApi.or(filter, columnFilter);
        }
    }
    return filter;
}

Example 28 with FilterPredicate

use of org.apache.parquet.filter2.predicate.FilterPredicate in project Gaffer by gchq.

the class QueryGeneratorTest method testQueryGeneratorForGetElementsWithEntitySeeds.

@Test
public void testQueryGeneratorForGetElementsWithEntitySeeds(@TempDir java.nio.file.Path tempDir) throws IOException, OperationException {
    // Given
    // - Create snapshot folder
    final String folder = String.format("file:///%s", tempDir.toString());
    final String snapshotFolder = folder + "/" + ParquetStore.getSnapshotPath(1000L);
    // - Write out Parquet files so know the partitioning
    CalculatePartitionerTest.writeData(snapshotFolder, new SchemaUtils(schema));
    // - Initialise store
    final ParquetStoreProperties storeProperties = new ParquetStoreProperties();
    storeProperties.setDataDir(folder);
    storeProperties.setTempFilesDir(folder + "/tmpdata");
    final ParquetStore store = (ParquetStore) ParquetStore.createStore("graphId", schema, storeProperties);
    // When 1 - no view, query for vertex 0
    GetElements getElements = new GetElements.Builder().input(new EntitySeed(0L)).seedMatching(SeedMatching.SeedMatchingType.RELATED).build();
    ParquetQuery query = new QueryGenerator(store).getParquetQuery(getElements);
    // Then 1
    final List expected = new ArrayList<>();
    final FilterPredicate vertex0 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 0L);
    final FilterPredicate source0 = eq(FilterApi.longColumn(ParquetStore.SOURCE), 0L);
    final FilterPredicate destination0 = eq(FilterApi.longColumn(ParquetStore.DESTINATION), 0L);
    for (final String group : Arrays.asList(TestGroups.ENTITY, TestGroups.ENTITY_2)) {
        final Path groupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(group, false));
        final Path pathForPartitionFile = new Path(groupFolderPath, ParquetStore.getFile(0));
        expected.add(new ParquetFileQuery(pathForPartitionFile, vertex0, true));
    }
    for (final String group : Arrays.asList(TestGroups.EDGE, TestGroups.EDGE_2)) {
        final Path groupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(group, false));
        final Path pathForPartitionFile = new Path(groupFolderPath, ParquetStore.getFile(0));
        expected.add(new ParquetFileQuery(pathForPartitionFile, source0, true));
        final Path reversedGroupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(group, true));
        final Path pathForReversedPartitionFile = new Path(reversedGroupFolderPath, ParquetStore.getFile(0));
        expected.add(new ParquetFileQuery(pathForReversedPartitionFile, destination0, true));
    }
    assertThat(expected).containsOnly(query.getAllParquetFileQueries().toArray());
    // When 2 - no view, query for vertices 0 and 1000000
    getElements = new GetElements.Builder().input(new EntitySeed(0L), new EntitySeed(1000000L)).seedMatching(SeedMatching.SeedMatchingType.RELATED).build();
    query = new QueryGenerator(store).getParquetQuery(getElements);
    // Then 2
    expected.clear();
    final FilterPredicate vertex1000000 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 1000000L);
    final FilterPredicate source1000000 = eq(FilterApi.longColumn(ParquetStore.SOURCE), 1000000L);
    final FilterPredicate destination1000000 = eq(FilterApi.longColumn(ParquetStore.DESTINATION), 1000000L);
    for (final String group : Arrays.asList(TestGroups.ENTITY, TestGroups.ENTITY_2)) {
        final Path groupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(group, false));
        final Path pathForPartitionFile1 = new Path(groupFolderPath, ParquetStore.getFile(0));
        expected.add(new ParquetFileQuery(pathForPartitionFile1, vertex0, true));
        final Path pathForPartitionFile2 = new Path(groupFolderPath, ParquetStore.getFile(9));
        expected.add(new ParquetFileQuery(pathForPartitionFile2, vertex1000000, true));
    }
    for (final String group : Arrays.asList(TestGroups.EDGE, TestGroups.EDGE_2)) {
        final Path groupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(group, false));
        final Path reversedGroupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(group, true));
        // Partition 0, vertex 0L
        final Path pathForPartitionFile1 = new Path(groupFolderPath, ParquetStore.getFile(0));
        expected.add(new ParquetFileQuery(pathForPartitionFile1, source0, true));
        // Partition 9, vertex 1000000L
        final Path pathForPartitionFile2 = new Path(groupFolderPath, ParquetStore.getFile(9));
        expected.add(new ParquetFileQuery(pathForPartitionFile2, source1000000, true));
        // Partition 0 of reversed, vertex 0L
        final Path pathForPartitionFile3 = new Path(reversedGroupFolderPath, ParquetStore.getFile(0));
        expected.add(new ParquetFileQuery(pathForPartitionFile3, destination0, true));
        // Partition 9 of reversed, vertex 1000000L
        final Path pathForPartitionFile4 = new Path(reversedGroupFolderPath, ParquetStore.getFile(9));
        expected.add(new ParquetFileQuery(pathForPartitionFile4, destination1000000, true));
    }
    assertThat(expected).containsOnly(query.getAllParquetFileQueries().toArray());
    // When 3 - view with filter that can be pushed down to Parquet, query for vertices 0 and 1000000
    getElements = new GetElements.Builder().input(new EntitySeed(0L), new EntitySeed(1000000L)).seedMatching(SeedMatching.SeedMatchingType.RELATED).view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().preAggregationFilter(new ElementFilter.Builder().select("count").execute(new IsMoreThan(10)).build()).build()).build()).build();
    query = new QueryGenerator(store).getParquetQuery(getElements);
    // Then 3
    expected.clear();
    final FilterPredicate source0AndCount = and(gt(FilterApi.intColumn("count"), 10), eq(FilterApi.longColumn(ParquetStore.SOURCE), 0L));
    final FilterPredicate source1000000AndCount = and(gt(FilterApi.intColumn("count"), 10), eq(FilterApi.longColumn(ParquetStore.SOURCE), 1000000L));
    final FilterPredicate destination0AndCount = and(gt(FilterApi.intColumn("count"), 10), eq(FilterApi.longColumn(ParquetStore.DESTINATION), 0L));
    final FilterPredicate destination1000000AndCount = and(gt(FilterApi.intColumn("count"), 10), eq(FilterApi.longColumn(ParquetStore.DESTINATION), 1000000L));
    final Path groupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(TestGroups.EDGE, false));
    final Path reversedGroupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(TestGroups.EDGE, true));
    // Partition 0, vertex 0L
    final Path pathForPartitionFile1 = new Path(groupFolderPath, ParquetStore.getFile(0));
    expected.add(new ParquetFileQuery(pathForPartitionFile1, source0AndCount, true));
    // Partition 9, vertex 1000000L
    final Path pathForPartitionFile2 = new Path(groupFolderPath, ParquetStore.getFile(9));
    expected.add(new ParquetFileQuery(pathForPartitionFile2, source1000000AndCount, true));
    // Partition 0 of reversed, vertex 0L
    final Path pathForPartitionFile3 = new Path(reversedGroupFolderPath, ParquetStore.getFile(0));
    expected.add(new ParquetFileQuery(pathForPartitionFile3, destination0AndCount, true));
    // Partition 9 of reversed, vertex 1000000L
    final Path pathForPartitionFile4 = new Path(reversedGroupFolderPath, ParquetStore.getFile(9));
    expected.add(new ParquetFileQuery(pathForPartitionFile4, destination1000000AndCount, true));
    assertThat(expected).containsOnly(query.getAllParquetFileQueries().toArray());
    // When 4 - view with filter that can't be pushed down to Parquet, query for vertices 0 and 1000000
    getElements = new GetElements.Builder().input(new EntitySeed(0L), new EntitySeed(1000000L)).seedMatching(SeedMatching.SeedMatchingType.RELATED).view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().preAggregationFilter(new ElementFilter.Builder().select("count").execute(new IsEvenFilter()).build()).build()).build()).build();
    query = new QueryGenerator(store).getParquetQuery(getElements);
    // Then 4
    expected.clear();
    // Partition 0, vertex 0L
    expected.add(new ParquetFileQuery(pathForPartitionFile1, source0, false));
    // Partition 9, vertex 1000000L
    expected.add(new ParquetFileQuery(pathForPartitionFile2, source1000000, false));
    // Partition 0 of reversed, vertex 0L
    expected.add(new ParquetFileQuery(pathForPartitionFile3, destination0, false));
    // Partition 9 of reversed, vertex 1000000L
    expected.add(new ParquetFileQuery(pathForPartitionFile4, destination1000000, false));
    assertThat(expected).containsOnly(query.getAllParquetFileQueries().toArray());
}

Example 29 with FilterPredicate

use of org.apache.parquet.filter2.predicate.FilterPredicate in project Gaffer by gchq.

the class QueryGeneratorTest method testQueryGeneratorForGetElementsWithEdgeSeeds.

@Test
public void testQueryGeneratorForGetElementsWithEdgeSeeds(@TempDir java.nio.file.Path tempDir) throws IOException, OperationException {
    // Given
    // - Create snapshot folder
    final String folder = String.format("file:///%s", tempDir.toString());
    final String snapshotFolder = folder + "/" + ParquetStore.getSnapshotPath(1000L);
    // - Write out Parquet files so know the partitioning
    CalculatePartitionerTest.writeData(snapshotFolder, new SchemaUtils(schema));
    // - Initialise store
    final ParquetStoreProperties storeProperties = new ParquetStoreProperties();
    storeProperties.setDataDir(folder);
    storeProperties.setTempFilesDir(folder + "/tmpdata");
    final ParquetStore store = (ParquetStore) ParquetStore.createStore("graphId", schema, storeProperties);
    // When 1 - no view, query for edges 0->1, 10--10000, 10000--10 with seed matching type set to EQUAL
    GetElements getElements = new GetElements.Builder().input(new EdgeSeed(0L, 1L, DirectedType.DIRECTED), new EdgeSeed(10L, 1000L, DirectedType.UNDIRECTED), new EdgeSeed(10000L, 10L, DirectedType.EITHER)).seedMatching(SeedMatching.SeedMatchingType.EQUAL).build();
    ParquetQuery query = new QueryGenerator(store).getParquetQuery(getElements);
    // Then 1
    final List expected = new ArrayList<>();
    final FilterPredicate source0 = eq(FilterApi.longColumn(ParquetStore.SOURCE), 0L);
    final FilterPredicate source10 = eq(FilterApi.longColumn(ParquetStore.SOURCE), 10L);
    final FilterPredicate source10000 = eq(FilterApi.longColumn(ParquetStore.SOURCE), 10000L);
    final FilterPredicate destination1 = eq(FilterApi.longColumn(ParquetStore.DESTINATION), 1L);
    final FilterPredicate destination10 = eq(FilterApi.longColumn(ParquetStore.DESTINATION), 10L);
    final FilterPredicate destination1000 = eq(FilterApi.longColumn(ParquetStore.DESTINATION), 1000L);
    final FilterPredicate directedTrue = eq(FilterApi.booleanColumn(ParquetStore.DIRECTED), true);
    final FilterPredicate directedFalse = eq(FilterApi.booleanColumn(ParquetStore.DIRECTED), false);
    final FilterPredicate source0Destination1DirectedTrue = and(and(source0, destination1), directedTrue);
    final FilterPredicate source10Destination1000DirectedFalse = and(and(source10, destination1000), directedFalse);
    final FilterPredicate source10000Destination10DirectedEither = and(source10000, destination10);
    for (final String group : Arrays.asList(TestGroups.EDGE, TestGroups.EDGE_2)) {
        final Path groupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(group, false));
        // 0->1 partition 0 of forward
        final Path pathForPartition0File = new Path(groupFolderPath, ParquetStore.getFile(0));
        // Comment here that don't need to look in the reversed directory
        expected.add(new ParquetFileQuery(pathForPartition0File, source0Destination1DirectedTrue, true));
        // 10--1000 partition 1 of forward
        final Path pathForPartition1File = new Path(groupFolderPath, ParquetStore.getFile(1));
        // Comment here that don't need to look in the reversed directory
        expected.add(new ParquetFileQuery(pathForPartition1File, source10Destination1000DirectedFalse, true));
        // 10000--10 partition 9 of forward
        final Path pathForPartition9File = new Path(groupFolderPath, ParquetStore.getFile(9));
        // Comment here that don't need to look in the reversed directory
        expected.add(new ParquetFileQuery(pathForPartition9File, source10000Destination10DirectedEither, true));
    }
    assertThat(expected).containsOnly(query.getAllParquetFileQueries().toArray());
    // When 2 - no view, query for edges 0->1, 10--10000, 10000--10 with seed matching type set to RELATED
    getElements = new GetElements.Builder().input(new EdgeSeed(0L, 1L, DirectedType.DIRECTED), new EdgeSeed(10L, 1000L, DirectedType.UNDIRECTED), new EdgeSeed(10000L, 10L, DirectedType.EITHER)).seedMatching(SeedMatching.SeedMatchingType.RELATED).build();
    query = new QueryGenerator(store).getParquetQuery(getElements);
    // Then 2
    expected.clear();
    final FilterPredicate vertex0 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 0L);
    final FilterPredicate vertex1 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 1L);
    final FilterPredicate vertex10 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 10L);
    final FilterPredicate vertex1000 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 1000L);
    final FilterPredicate vertex10000 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 10000L);
    final FilterPredicate vertex0or1 = or(vertex0, vertex1);
    final FilterPredicate vertex10or1000 = or(vertex10, vertex1000);
    final FilterPredicate vertex10000or10 = or(vertex10000, vertex10);
    for (final String group : Arrays.asList(TestGroups.ENTITY, TestGroups.ENTITY_2)) {
        final Path groupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(group, false));
        // 0 and 1 in partition 0
        final Path pathForPartition0File = new Path(groupFolderPath, ParquetStore.getFile(0));
        expected.add(new ParquetFileQuery(pathForPartition0File, vertex0or1, true));
        // 10 or 1000 and 10000 or 10 in partition 1 (NB 1000 and 10000 't appear in partition 1 but this doesn't cause any incorrect results, and will be fixed in later versions)
        final Path pathForPartition1File = new Path(groupFolderPath, ParquetStore.getFile(1));
        expected.add(new ParquetFileQuery(pathForPartition1File, or(vertex10or1000, vertex10000or10), true));
        // 10 or 1000 and 1000 or 10000 in partition 9
        final Path pathForPartition9File = new Path(groupFolderPath, ParquetStore.getFile(9));
        expected.add(new ParquetFileQuery(pathForPartition9File, or(vertex10or1000, vertex10000or10), true));
    }
    for (final String group : Arrays.asList(TestGroups.EDGE, TestGroups.EDGE_2)) {
        final Path groupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(group, false));
        // 0->1 partition 0 of forward
        final Path pathForPartition0File = new Path(groupFolderPath, ParquetStore.getFile(0));
        // Comment here that don't need to look in the reversed directory
        expected.add(new ParquetFileQuery(pathForPartition0File, source0Destination1DirectedTrue, true));
        // 10--1000 partition 1 of forward
        final Path pathForPartition1File = new Path(groupFolderPath, ParquetStore.getFile(1));
        // Comment here that don't need to look in the reversed directory
        expected.add(new ParquetFileQuery(pathForPartition1File, source10Destination1000DirectedFalse, true));
        // 10000--10 partition 9 of forward
        final Path pathForPartition9File = new Path(groupFolderPath, ParquetStore.getFile(9));
        // Comment here that don't need to look in the reversed directory
        expected.add(new ParquetFileQuery(pathForPartition9File, source10000Destination10DirectedEither, true));
    }
    assertThat(expected).containsOnly(query.getAllParquetFileQueries().toArray());
}

Example 30 with FilterPredicate

use of org.apache.parquet.filter2.predicate.FilterPredicate in project Gaffer by gchq.

the class QueryGenerator method getPredicateFromView.

// TODO raise issue saying that could optimise so that only the filters that have not been fully applied
// are reapplied, and it should be able to return the fact that all filters have been applied
// Either the result is:
// - a filter and that fully applies the view
// - a filter and that doesn't fully apply the view
// - no filter and that doesn't fully apply the view
// - no filter and that fully applies the view
// Boolean indicates whether logic was fully applied
private Pair<FilterPredicate, Boolean> getPredicateFromView(final View view, final String group, final boolean isEntityGroup) throws SerialisationException, OperationException {
    if (null == view) {
        return new Pair<>(null, true);
    }
    final ViewElementDefinition ved = view.getElement(group);
    FilterPredicate filterPredicate = null;
    boolean fullyAppliedInAll = true;
    if (null != ved) {
        List<TupleAdaptedPredicate<String, ?>> preAggFilterFunctions = ved.getPreAggregationFilterFunctions();
        if (null != preAggFilterFunctions) {
            for (final TupleAdaptedPredicate<String, ?> filterFunctionContext : preAggFilterFunctions) {
                final JavaPredicateToParquetPredicate predicateConverter = new JavaPredicateToParquetPredicate(schemaUtils, filterFunctionContext.getPredicate(), filterFunctionContext.getSelection(), group);
                filterPredicate = FilterPredicateUtils.and(filterPredicate, predicateConverter.getParquetPredicate());
                if (!predicateConverter.isFullyApplied()) {
                    fullyAppliedInAll = false;
                }
            }
        }
    }
    return new Pair<>(filterPredicate, fullyAppliedInAll);
}