Examples with FDBRecordContext com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext used on opensource projects

Example 51 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class TextIndexTest method saveSimpleDocuments.

@Test
public void saveSimpleDocuments() throws Exception {
    final SimpleDocument simpleDocument = SimpleDocument.newBuilder().setDocId(1066L).setText("This is a simple document. There isn't much going on here, if I'm honest.").setGroup(0).build();
    final SimpleDocument buffaloDocument = SimpleDocument.newBuilder().setDocId(1415L).setText("Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo Buffalo buffalo buffalo.").setGroup(1).build();
    final SimpleDocument shakespeareDocument = SimpleDocument.newBuilder().setDocId(1623L).setText(TextSamples.ROMEO_AND_JULIET_PROLOGUE).setGroup(2).build();
    final SimpleDocument noTextDocument = SimpleDocument.newBuilder().setDocId(0L).setGroup(0).build();
    final SimpleDocument emptyDocument = SimpleDocument.newBuilder().setDocId(1L).setGroup(1).setText("").build();
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        Index index = recordStore.getRecordMetaData().getIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME);
        recordStore.saveRecord(simpleDocument);
        final int firstKeys = getSaveIndexKeyCount(recordStore);
        assertEquals(simpleDocument.getText().split(" ").length, firstKeys);
        final int firstKeyBytesWritten = getSaveIndexKeyBytes(recordStore);
        final int firstValueBytesWritten = getSaveIndexValueBytes(recordStore);
        List<Map.Entry<Tuple, List<Integer>>> entryList = scanMapEntries(recordStore, index, Tuple.from("document"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1066L), Collections.singletonList(4))), entryList);
        resetTimer(recordStore);
        recordStore.saveRecord(buffaloDocument);
        final int secondKeys = getSaveIndexKeyCount(recordStore);
        assertEquals(1, secondKeys);
        entryList = scanMapEntries(recordStore, index, Tuple.from("buffalo"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1415L), IntStream.range(0, 11).boxed().collect(Collectors.toList()))), entryList);
        resetTimer(recordStore);
        recordStore.saveRecord(shakespeareDocument);
        final int thirdKeys = getSaveIndexKeyCount(recordStore);
        assertEquals(82, thirdKeys);
        final int thirdBytesWritten = getSaveIndexKeyBytes(recordStore) + getSaveIndexValueBytes(recordStore);
        entryList = scanMapEntries(recordStore, index, Tuple.from("parents"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1623L), Arrays.asList(57, 72))), entryList);
        entryList = toMapEntries(scanIndex(recordStore, index, TupleRange.prefixedBy("h")), null);
        assertEquals(Arrays.asList(entryOf(Tuple.from("hands", 1623), Collections.singletonList(26)), entryOf(Tuple.from("here", 1066), Collections.singletonList(10)), entryOf(Tuple.from("here", 1623), Collections.singletonList(101)), entryOf(Tuple.from("honest", 1066), Collections.singletonList(13)), entryOf(Tuple.from("hours", 1623), Collections.singletonList(87)), entryOf(Tuple.from("households", 1623), Collections.singletonList(1))), entryList);
        List<Message> recordList = recordStore.scanIndexRecords(index.getName(), BY_TEXT_TOKEN, TupleRange.prefixedBy("h"), null, ScanProperties.FORWARD_SCAN).map(FDBIndexedRecord::getRecord).asList().get();
        assertEquals(Arrays.asList(shakespeareDocument, simpleDocument, shakespeareDocument, simpleDocument, shakespeareDocument, shakespeareDocument), recordList);
        resetTimer(recordStore);
        recordStore.saveRecord(noTextDocument);
        assertEquals(0, getSaveIndexKeyCount(recordStore));
        assertEquals(0, getLoadIndexKeyCount(recordStore));
        resetTimer(recordStore);
        recordStore.saveRecord(emptyDocument);
        assertEquals(0, getSaveIndexKeyCount(recordStore));
        assertEquals(0, getLoadIndexKeyCount(recordStore));
        resetTimer(recordStore);
        recordStore.deleteRecord(Tuple.from(1623L));
        // all deleted but four overlaps with first record
        assertEquals(thirdKeys - 4, getDeleteIndexKeyCount(recordStore));
        // four keys of overlap overwritten
        assertEquals(4, getSaveIndexKeyCount(recordStore));
        assertThat(getDeleteIndexKeyBytes(recordStore) + getDeleteIndexValueBytes(recordStore), allOf(greaterThan(thirdKeys - 1), lessThan(thirdBytesWritten)));
        entryList = scanMapEntries(recordStore, index, Tuple.from("parents"));
        assertEquals(Collections.emptyList(), entryList);
        resetTimer(recordStore);
        recordStore.saveRecord(simpleDocument.toBuilder().setDocId(1707L).build());
        assertEquals(firstKeys * 2, getLoadIndexKeyCount(recordStore));
        assertEquals(firstKeys, getSaveIndexKeyCount(recordStore));
        // should overwrite all the same keys
        assertEquals(firstKeyBytesWritten, getSaveIndexKeyBytes(recordStore));
        final int seventhValueBytesWritten = getSaveIndexValueBytes(recordStore);
        // contains same info as first value bytes + extra keys, but not key prefixes
        assertThat(seventhValueBytesWritten, allOf(greaterThan(firstValueBytesWritten), lessThan(firstKeyBytesWritten + firstValueBytesWritten)));
        entryList = scanMapEntries(recordStore, index, Tuple.from("document"));
        assertEquals(Arrays.asList(entryOf(Tuple.from(1066L), Collections.singletonList(4)), entryOf(Tuple.from(1707L), Collections.singletonList(4))), entryList);
        resetTimer(recordStore);
        recordStore.deleteRecord(Tuple.from(1066L));
        assertEquals(firstKeys, getLoadIndexKeyCount(recordStore));
        // each of the original keys are deleted
        assertEquals(firstKeys, getDeleteIndexKeyCount(recordStore));
        assertEquals(firstKeyBytesWritten, getDeleteIndexKeyBytes(recordStore));
        assertEquals(firstValueBytesWritten + seventhValueBytesWritten, getDeleteIndexValueBytes(recordStore));
        // a new set of keys are all written
        assertEquals(firstKeys, getSaveIndexKeyCount(recordStore));
        // they should have the same size (though their contents are different)
        assertEquals(firstKeyBytesWritten, getSaveIndexKeyBytes(recordStore));
        assertEquals(firstValueBytesWritten, getSaveIndexValueBytes(recordStore));
        entryList = scanMapEntries(recordStore, index, Tuple.from("document"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1707L), Collections.singletonList(4))), entryList);
        commit(context);
    }
}

Example 52 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class TextIndexTest method querySimpleDocuments.

@Test
public void querySimpleDocuments() throws Exception {
    final List<SimpleDocument> documents = TextIndexTestUtils.toSimpleDocuments(Arrays.asList(TextSamples.ANGSTROM, TextSamples.AETHELRED, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.YIDDISH, TextSamples.CHINESE_SIMPLIFIED, TextSamples.KOREAN, "a b a b a b c"));
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        documents.forEach(recordStore::saveRecord);
        // Contains
        assertEquals(Arrays.asList(0L, 1L, 2L), querySimpleDocumentsWithIndex(Query.field("text").text().contains("the"), 329921958, true));
        assertEquals(Collections.singletonList(0L), querySimpleDocumentsWithIndex(Query.field("text").text().contains("angstrom"), -1859676822, true));
        assertEquals(Collections.emptyList(), querySimpleDocumentsWithIndex(Query.field("text").text().contains("Ångström"), 2028628575, true));
        assertEquals(Collections.singletonList(3L), querySimpleDocumentsWithIndex(Query.field("text").text().contains("שפראך"), 1151275308, true));
        // Contains all
        assertEquals(Collections.singletonList(0L), querySimpleDocumentsWithIndex(Query.field("text").text().containsAll("Ångström"), 1999999424, true));
        assertEquals(Collections.emptyList(), querySimpleDocumentsWithIndex(Query.field("text").text().containsAll(Collections.singletonList("Ångström")), 2028628575, true));
        assertEquals(Collections.singletonList(0L), querySimpleDocumentsWithIndex(Query.field("text").text().containsAll("the angstrom"), 865061914, true));
        assertEquals(Collections.singletonList(0L), querySimpleDocumentsWithIndex(Query.field("text").text().containsAll(Arrays.asList("the", "angstrom")), 4380219, true));
        assertEquals(Collections.singletonList(0L), querySimpleDocumentsWithIndex(Query.field("text").text().containsAll(Arrays.asList("", "angstrom")), -1000802292, true));
        assertEquals(Collections.singletonList(5L), querySimpleDocumentsWithIndex(Query.field("text").text().containsAll("한국어를"), -1046915537, true));
        // Contains all within a distance
        assertEquals(Collections.singletonList(0L), querySimpleDocumentsWithIndex(Query.field("text").text().containsAll("Ångström named", 4), -1408252035, true));
        assertEquals(Collections.singletonList(0L), querySimpleDocumentsWithIndex(Query.field("text").text().containsAll("Ångström named", 3), -1408252996, true));
        assertEquals(Collections.emptyList(), querySimpleDocumentsWithIndex(Query.field("text").text().containsAll("Ångström named", 2), -1408253957, true));
        assertEquals(Collections.emptyList(), querySimpleDocumentsWithIndex(Query.field("text").text().containsAll(Arrays.asList("Ångström", "named"), 4), -2041874864, true));
        assertEquals(Collections.singletonList(6L), querySimpleDocumentsWithIndex(Query.field("text").text().containsAll("a c", 2), 2135218554, true));
        assertEquals(Collections.emptyList(), querySimpleDocumentsWithIndex(Query.field("text").text().containsAll("a c", 1), 2135217593, true));
        assertEquals(Collections.singletonList(6L), querySimpleDocumentsWithIndex(Query.field("text").text().containsAll("b c", 2), -416938407, true));
        assertEquals(Collections.singletonList(6L), querySimpleDocumentsWithIndex(Query.field("text").text().containsAll("b c", 1), -416939368, true));
        // Contains any
        assertEquals(Collections.singletonList(0L), querySimpleDocumentsWithIndex(Query.field("text").text().containsAny("Ångström"), -147781547, true));
        assertEquals(Collections.emptyList(), querySimpleDocumentsWithIndex(Query.field("text").text().containsAny(Collections.singletonList("Ångström")), -119152396, true));
        assertEquals(Arrays.asList(0L, 1L, 2L), querySimpleDocumentsWithIndex(Query.field("text").text().containsAny("the angstrom"), -1282719057, true));
        assertEquals(Arrays.asList(0L, 1L, 2L), querySimpleDocumentsWithIndex(Query.field("text").text().containsAny(Arrays.asList("the", "angstrom")), -2143400752, true));
        // Contains phrase
        assertEquals(Collections.singletonList(2L), querySimpleDocumentsWithIndex(Query.field("text").text().containsPhrase("Civil blood makes. Civil hands unclean"), -993768059, true));
        assertEquals(Collections.singletonList(2L), querySimpleDocumentsWithIndex(Query.field("text").text().containsPhrase(Arrays.asList("civil", "blood", "makes", "civil", "", "unclean")), 1855137352, true));
        assertEquals(Collections.emptyList(), querySimpleDocumentsWithIndex(Query.field("text").text().containsPhrase(Arrays.asList("Civil", "blood", "makes", "civil", "", "unclean")), 853144168, true));
        assertEquals(Collections.singletonList(2L), querySimpleDocumentsWithIndex(Query.field("text").text().containsPhrase(Arrays.asList("", "civil", "blood", "makes", "civil", "", "unclean", "")), 930039198, true));
        assertEquals(Collections.singletonList(6L), querySimpleDocumentsWithIndex(Query.field("text").text().containsPhrase("a b a b c"), -623744405, true));
        // Contains prefix
        assertEquals(Arrays.asList(2L, 0L, 1L), querySimpleDocumentsWithIndex(Query.field("text").text().containsPrefix("un"), 1067159426, true));
        assertEquals(Collections.singletonList(3L), querySimpleDocumentsWithIndex(Query.field("text").text().containsPrefix("א"), -1009839303, true));
        assertEquals(Collections.singletonList(4L), querySimpleDocumentsWithIndex(Query.field("text").text().containsPrefix("苹果"), -1529274452, true));
        assertEquals(Collections.singletonList(5L), querySimpleDocumentsWithIndex(Query.field("text").text().containsPrefix(Normalizer.normalize("한국", Normalizer.Form.NFKD)), -1860545817, true));
        assertEquals(Collections.singletonList(5L), // note that the second character is only 2 of the 3 Jamo components
        querySimpleDocumentsWithIndex(Query.field("text").text().containsPrefix("한구"), 1377518291, true));
        // Contains any prefix
        assertEquals(ImmutableSet.of(0L, 1L, 2L, 3L), new HashSet<>(querySimpleDocumentsWithIndex(Query.field("text").text().containsAnyPrefix("civ א un"), 1227233680, true)));
        assertEquals(ImmutableSet.of(0L, 1L, 2L, 3L), new HashSet<>(querySimpleDocumentsWithIndex(Query.field("text").text().containsAnyPrefix("cIv ַא Un"), -794472473, true)));
        assertEquals(ImmutableSet.of(0L, 1L, 2L, 3L), new HashSet<>(querySimpleDocumentsWithIndex(Query.field("text").text().containsAnyPrefix(Arrays.asList("civ", "א", "un")), 1486849487, true)));
        assertEquals(ImmutableSet.of(2L), new HashSet<>(querySimpleDocumentsWithIndex(Query.field("text").text().containsAnyPrefix(Arrays.asList("civ", "אַ", "Un")), 1905505336, true)));
        // Contains all prefixes
        assertEquals(Collections.singletonList(2L), querySimpleDocumentsWithIndex(Query.field("text").text().containsAllPrefixes("civ un"), 1757831895, false));
        assertEquals(Collections.singletonList(2L), querySimpleDocumentsWithIndex(Query.field("text").text().containsAllPrefixes("civ un", false), -900079353, true));
        assertEquals(ImmutableSet.of(0L, 1L), new HashSet<>(querySimpleDocumentsWithIndex(Query.field("text").text().containsAllPrefixes("wa th"), -1203466155, false)));
        assertEquals(ImmutableSet.of(0L, 1L), new HashSet<>(querySimpleDocumentsWithIndex(Query.field("text").text().containsAllPrefixes("wa th", false), -433119192, true)));
        commit(context);
    }
}

Example 53 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class TextIndexTest method scanWithContinuations.

private void scanWithContinuations(@Nonnull Index index, @Nonnull String token, int limit, boolean reverse) throws Exception {
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        final List<IndexEntry> firstResults = scanIndex(recordStore, index, TupleRange.allOf(Tuple.from(token)));
        validateSorted(firstResults);
        List<IndexEntry> scanResults = new ArrayList<>(firstResults.size());
        byte[] continuation = null;
        do {
            RecordCursorIterator<IndexEntry> cursor = recordStore.scanIndex(index, BY_TEXT_TOKEN, TupleRange.allOf(Tuple.from(token)), continuation, reverse ? ScanProperties.REVERSE_SCAN : ScanProperties.FORWARD_SCAN).asIterator();
            for (int i = 0; i < limit || limit == 0; i++) {
                if (cursor.hasNext()) {
                    scanResults.add(cursor.next());
                } else {
                    break;
                }
            }
            continuation = cursor.getContinuation();
            // fire off but don't wait for an on-has-next
            CompletableFuture<Boolean> hasNextFuture = cursor.onHasNext();
            // wait for the same on-has-next
            assertEquals(hasNextFuture.get(), cursor.hasNext());
            if (!cursor.hasNext()) {
                assertThat(cursor.getNoNextReason().isSourceExhausted(), is(true));
            }
        } while (continuation != null);
        if (reverse) {
            Collections.reverse(scanResults);
        }
        assertEquals(firstResults, scanResults);
    }
}

Example 54 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method readVersionFromStoredRecordInTwoStores.

/**
 * Store a record in one store, then store a different record in a different store with the same primary key
 * and validate that the version read for the first record matches the version written and not the version
 * for the second record (which could happen if the local version cache leaks information between stores).
 */
@ParameterizedTest(name = "readVersionFromStoredRecordInTwoStores [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void readVersionFromStoredRecordInTwoStores(int testFormatVersion, boolean testSplitLongRecords) {
    formatVersion = testFormatVersion;
    splitLongRecords = testSplitLongRecords;
    final MySimpleRecord record1 = MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(42).build();
    final MySimpleRecord record2 = MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(1729).build();
    final FDBRecordVersion version1;
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        FDBStoredRecord<?> storedRecord1 = recordStore.saveRecord(record1);
        assertNotNull(storedRecord1.getVersion());
        context.commit();
        assertNotNull(context.getVersionStamp());
        version1 = storedRecord1.getVersion().withCommittedVersion(context.getVersionStamp());
    }
    final FDBRecordVersion version2;
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        final FDBRecordStore recordStore2 = recordStore.asBuilder().setSubspace(subspace2).create();
        FDBStoredRecord<?> storedRecord2 = recordStore2.saveRecord(record2);
        assertNotNull(storedRecord2.getVersion());
        FDBStoredRecord<?> storedRecord1 = recordStore.loadRecord(Tuple.from(record1.getRecNo()));
        assertNotNull(storedRecord1);
        assertEquals(version1, storedRecord1.getVersion());
        context.commit();
        assertNotNull(context.getVersionStamp());
        version2 = storedRecord2.getVersion().withCommittedVersion(context.getVersionStamp());
    }
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        final FDBRecordStore recordStore2 = recordStore.asBuilder().setSubspace(subspace2).open();
        FDBStoredRecord<?> storedRecord1 = recordStore.loadRecord(Tuple.from(record1.getRecNo()));
        assertNotNull(storedRecord1);
        assertEquals(version1, storedRecord1.getVersion());
        assertEquals(record1, storedRecord1.getRecord());
        FDBStoredRecord<?> storedRecord2 = recordStore2.loadRecord(Tuple.from(record2.getRecNo()));
        assertNotNull(storedRecord2);
        assertEquals(version2, storedRecord2.getVersion());
        assertEquals(record2, storedRecord2.getRecord());
    }
}

Example 55 with FDBRecordContext

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method assertMaxVersionsForGroups.

@SuppressWarnings("try")
private void assertMaxVersionsForGroups(@Nonnull SortedMap<Integer, FDBRecordVersion> groupsToVersions) {
    try (FDBRecordContext context = openContext(maxEverVersionWithGroupingHook)) {
        Index index = metaData.getIndex("max_ever_version_with_grouping");
        List<IndexEntry> entries = new ArrayList<>(groupsToVersions.size());
        for (Map.Entry<Integer, FDBRecordVersion> mapEntry : groupsToVersions.entrySet()) {
            entries.add(new IndexEntry(index, Key.Evaluated.scalar(mapEntry.getKey()), Key.Evaluated.scalar(mapEntry.getValue())));
        }
        assertMaxVersionEntries(index, entries);
    }
}