Examples with ExecuteProperties com.apple.foundationdb.record.ExecuteProperties used on opensource projects

Example 6 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class IndexingByRecords method buildEndpoints.

@Nonnull
private CompletableFuture<TupleRange> buildEndpoints(@Nonnull FDBRecordStore store, @Nonnull RangeSet rangeSet, @Nullable AtomicLong recordsScanned) {
    boolean isIdempotent = store.getIndexMaintainer(common.getIndex()).isIdempotent();
    final IsolationLevel isolationLevel = isIdempotent ? // before this method's query) will be re-indexed.
    IsolationLevel.SNAPSHOT : IsolationLevel.SERIALIZABLE;
    final ExecuteProperties limit1 = ExecuteProperties.newBuilder().setReturnedRowLimit(1).setIsolationLevel(isolationLevel).build();
    final ScanProperties forward = new ScanProperties(limit1);
    RecordCursor<FDBStoredRecord<Message>> beginCursor = store.scanRecords(recordsRange, null, forward);
    CompletableFuture<Tuple> begin = beginCursor.onNext().thenCompose(result -> {
        if (result.hasNext()) {
            Tuple firstTuple = result.get().getPrimaryKey();
            return buildRange(store, null, firstTuple, recordsScanned).thenApply(vignore -> firstTuple);
        } else {
            // Empty range -- add the whole thing.
            return rangeSet.insertRange(store.ensureContextActive(), null, null).thenApply(bignore -> null);
        }
    });
    final ScanProperties backward = new ScanProperties(limit1, true);
    RecordCursor<FDBStoredRecord<Message>> endCursor = store.scanRecords(recordsRange, null, backward);
    CompletableFuture<Tuple> end = endCursor.onNext().thenCompose(result -> {
        if (result.hasNext()) {
            Tuple lastTuple = result.get().getPrimaryKey();
            return buildRange(store, lastTuple, null, recordsScanned).thenApply(vignore -> lastTuple);
        } else {
            // by the above future, so this has nothing to do.
            return CompletableFuture.completedFuture(null);
        }
    });
    return begin.thenCombine(end, (firstTuple, lastTuple) -> {
        if (firstTuple == null || firstTuple.equals(lastTuple)) {
            return null;
        } else {
            return new TupleRange(firstTuple, lastTuple, EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_EXCLUSIVE);
        }
    });
}

Example 7 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class TextIndexTest method performQueryWithIndexScan.

@Nonnull
private Set<Long> performQueryWithIndexScan(@Nonnull RecordMetaDataHook hook, @Nonnull Index index, @Nonnull QueryComponent filter) throws Exception {
    final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setTimeLimit(3000).build();
    final RecordQuery query = RecordQuery.newBuilder().setRecordType(SIMPLE_DOC).setRequiredResults(Collections.singletonList(field("doc_id"))).setFilter(filter).build();
    Set<Long> results = new HashSet<>();
    RecordQueryPlan plan;
    byte[] continuation;
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        RecordQueryPlanner planner = new RecordQueryPlanner(recordStore.getRecordMetaData(), recordStore.getRecordStoreState());
        plan = planner.plan(query);
        assertThat(filter, instanceOf(FieldWithComparison.class));
        FieldWithComparison fieldFilter = (FieldWithComparison) filter;
        if (fieldFilter.getComparison() instanceof Comparisons.TextContainsAllPrefixesComparison && ((Comparisons.TextContainsAllPrefixesComparison) fieldFilter.getComparison()).isStrict()) {
            // Strict field field comparisons cannot be covering
            assertThat(plan, descendant(textIndexScan(indexName(index.getName()))));
        } else {
            assertThat(plan, descendant(coveringIndexScan(textIndexScan(indexName(index.getName())))));
        }
        try (RecordCursor<Long> cursor = recordStore.executeQuery(plan, null, executeProperties).map(record -> record.getPrimaryKey().getLong(0))) {
            cursor.forEach(results::add).get();
            continuation = cursor.getNext().getContinuation().toBytes();
        }
    }
    while (continuation != null) {
        try (FDBRecordContext context = openContext()) {
            openRecordStore(context, hook);
            try (RecordCursor<Long> cursor = recordStore.executeQuery(plan, continuation, executeProperties).map(record -> record.getPrimaryKey().getLong(0))) {
                cursor.forEach(results::add).get();
                continuation = cursor.getNext().getContinuation().toBytes();
            }
        }
    }
    return results;
}

Example 8 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class TextIndexTest method queryDocumentsWithScanLimit.

@Test
public void queryDocumentsWithScanLimit() throws Exception {
    // Load a big (ish) data set
    final int recordCount = 100;
    final int batchSize = 10;
    for (int i = 0; i < recordCount; i += batchSize) {
        try (FDBRecordContext context = openContext()) {
            openRecordStore(context);
            for (int j = 0; j < batchSize; j++) {
                SimpleDocument document = SimpleDocument.newBuilder().setDocId(i + j).setText((i + j) % 2 == 0 ? "some" : "text").build();
                recordStore.saveRecord(document);
            }
            commit(context);
        }
    }
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        RecordQuery query = RecordQuery.newBuilder().setRecordType(SIMPLE_DOC).setFilter(Query.field("text").text().containsAll("some text")).build();
        RecordQueryPlan plan = planner.plan(query);
        boolean done = false;
        int totalKeysLoaded = 0;
        byte[] continuation = null;
        while (!done) {
            final int priorKeysLoaded = getLoadTextEntryCount(recordStore);
            ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedRecordsLimit(50).build();
            RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, executeProperties);
            assertEquals(Collections.emptyList(), cursor.asList().get());
            RecordCursorResult<FDBQueriedRecord<Message>> noNextResult = cursor.getNext();
            assertThat(noNextResult.hasNext(), is(false));
            final int newKeysLoaded = getLoadTextEntryCount(recordStore);
            totalKeysLoaded += newKeysLoaded - priorKeysLoaded;
            if (!noNextResult.getNoNextReason().isSourceExhausted()) {
                assertEquals(50, newKeysLoaded - priorKeysLoaded);
                assertEquals(RecordCursor.NoNextReason.SCAN_LIMIT_REACHED, noNextResult.getNoNextReason());
                assertNotNull(noNextResult.getContinuation().toBytes());
            } else {
                assertNull(noNextResult.getContinuation().toBytes());
                done = true;
            }
            continuation = noNextResult.getContinuation().toBytes();
        }
        assertEquals(recordCount + 2, totalKeysLoaded);
        commit(context);
    }
}

Example 9 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class GeophileQueryTest method testDistance.

@Test
@Tag(Tags.Slow)
public void testDistance() throws Exception {
    final RecordMetaDataHook hook = md -> {
        md.addIndex("City", CITY_LOCATION_COVERING_INDEX);
    };
    loadCities(hook, 0);
    final int centerId = 5391959;
    final double distance = 1;
    final int scanLimit = 5000;
    final RecordQueryPlan scanPlan = distanceFilterScan(distance);
    final Set<Integer> scanResults = new HashSet<>();
    byte[] continuation = null;
    do {
        try (FDBRecordContext context = openContext()) {
            openRecordStore(context, hook);
            EvaluationContext joinContext = bindCenter(centerId);
            ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedRecordsLimit(scanLimit).build();
            RecordCursor<FDBQueriedRecord<Message>> recordCursor = scanPlan.execute(recordStore, joinContext, continuation, executeProperties);
            recordCursor.forEach(city -> {
                TestRecordsGeoProto.City.Builder cityBuilder = TestRecordsGeoProto.City.newBuilder().mergeFrom(city.getRecord());
                LOGGER.debug(String.format("Scan found %s: %s", cityBuilder.getGeoNameId(), cityBuilder.getName()));
                scanResults.add(cityBuilder.getGeoNameId());
            }).join();
            continuation = recordCursor.getNext().getContinuation().toBytes();
            commit(context);
        }
    } while (continuation != null);
    final RecordQueryPlan indexPlan = distanceSpatialQuery(distance, false);
    final Set<Integer> indexResults = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        RecordCursor<FDBQueriedRecord<Message>> recordCursor = indexPlan.execute(recordStore, bindCenter(centerId));
        recordCursor.forEach(city -> {
            TestRecordsGeoProto.City.Builder cityBuilder = TestRecordsGeoProto.City.newBuilder().mergeFrom(city.getRecord());
            LOGGER.debug(String.format("Index found %s: %s", cityBuilder.getGeoNameId(), cityBuilder.getName()));
            indexResults.add(cityBuilder.getGeoNameId());
        }).join();
        int given = timer.getCount(FDBStoreTimer.Counts.QUERY_FILTER_GIVEN);
        int passed = timer.getCount(FDBStoreTimer.Counts.QUERY_FILTER_PASSED);
        int discarded = timer.getCount(FDBStoreTimer.Counts.QUERY_DISCARDED);
        assertThat("Should have passed more than discarded", passed, greaterThan(discarded));
        commit(context);
    }
    final RecordQueryPlan coveringPlan = distanceSpatialQuery(distance, true);
    final Set<Integer> coveringResults = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        RecordCursor<FDBQueriedRecord<Message>> recordCursor = indexPlan.execute(recordStore, bindCenter(centerId));
        recordCursor.forEach(city -> {
            TestRecordsGeoProto.City.Builder cityBuilder = TestRecordsGeoProto.City.newBuilder().mergeFrom(city.getRecord());
            LOGGER.debug(String.format("Covering found %s: %s", cityBuilder.getGeoNameId(), cityBuilder.getName()));
            coveringResults.add(cityBuilder.getGeoNameId());
        }).join();
        commit(context);
    }
    assertEquals(scanResults, indexResults);
    assertEquals(scanResults, coveringResults);
}

Example 10 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class QueryPlanCursorTest method compareSkipsAndCursors.

private void compareSkipsAndCursors(RecordQueryPlan plan, int amount) throws Exception {
    final Function<FDBQueriedRecord<Message>, Long> getRecNo = r -> {
        TestRecords1Proto.MySimpleRecord.Builder record = TestRecords1Proto.MySimpleRecord.newBuilder();
        record.mergeFrom(r.getRecord());
        return record.getRecNo();
    };
    final ExecuteProperties justLimit = ExecuteProperties.newBuilder().setReturnedRowLimit(amount).build();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        final List<Long> whole;
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan)) {
            whole = cursor.map(getRecNo).asList().get();
        }
        assertTrue(whole.size() > amount, "should have more than one batch");
        final List<Long> byCursors = new ArrayList<>();
        byte[] continuation = null;
        do {
            try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, justLimit)) {
                RecordCursorResult<FDBQueriedRecord<Message>> result = null;
                do {
                    result = cursor.getNext();
                    if (result.hasNext()) {
                        byCursors.add(getRecNo.apply(result.get()));
                    }
                } while (result.hasNext());
                continuation = result.getContinuation().toBytes();
            }
        } while (continuation != null);
        assertEquals(whole, byCursors);
        final List<Long> byOffsets = new ArrayList<>();
        int offset = 0;
        while (true) {
            final ExecuteProperties skipAndLimit = ExecuteProperties.newBuilder().setSkip(offset).setReturnedRowLimit(amount).setIsolationLevel(justLimit.getIsolationLevel()).build();
            try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, skipAndLimit)) {
                final List<Long> next = cursor.map(getRecNo).asList().get();
                byOffsets.addAll(next);
                if (next.size() < amount) {
                    break;
                } else {
                    offset += next.size();
                }
            }
        }
        assertEquals(whole, byOffsets);
    }
}