Examples with IndexEntry com.apple.foundationdb.record.IndexEntry used on opensource projects

Example 46 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreReplaceIndexTest method markExistingAsReplacement.

@Test
public void markExistingAsReplacement() {
    final String recordTypeName = "MyOtherRecord";
    final Index origIndex = new Index("MyOtherRecord$(num_value_2, num_value_3_indexed)", "num_value_2", "num_value_3_indexed");
    final Index newIndex = new Index("MyOtherRecord$(num_value_3_indexed, num_value_2)", "num_value_3_indexed", "num_value_2");
    final RecordMetaDataHook addBothIndexes = composeHooks(addIndexHook(recordTypeName, origIndex), addIndexHook(recordTypeName, newIndex));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, addBothIndexes);
        assertTrue(recordStore.isIndexReadable(origIndex), "Old index should be readable at start");
        assertTrue(recordStore.isIndexReadable(newIndex), "New index should be readable at start");
        recordStore.saveRecord(TestRecords1Proto.MyOtherRecord.newBuilder().setRecNo(1415L).setNumValue2(42).setNumValue3Indexed(3).build());
        final List<IndexEntry> oldIndexEntries = scanIndex(origIndex);
        assertThat(oldIndexEntries, hasSize(1));
        assertEquals(Tuple.from(1415L), oldIndexEntries.get(0).getPrimaryKey());
        assertEquals(Tuple.from(42L, 3L, 1415L), oldIndexEntries.get(0).getKey());
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, composeHooks(addIndexAndReplacements(recordTypeName, origIndex, newIndex), bumpMetaDataVersionHook()));
        assertTrue(recordStore.isIndexDisabled(origIndex.getName()));
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, composeHooks(addBothIndexes, bumpMetaDataVersionHook(), bumpMetaDataVersionHook()));
        assertTrue(uncheckedMarkIndexReadable(origIndex));
        assertThat(scanIndex(origIndex), empty());
    }
}

Example 47 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreReplaceIndexTest method buildTwoReplacements.

@Test
public void buildTwoReplacements() {
    final String recordTypeName = "MySimpleRecord";
    final Index origIndex = new Index("MySimpleRecord$(str_value_indexed, num_value_2)", "str_value_indexed", "num_value_2");
    final Index newIndex1 = new Index("MySimpleRecord$(str_value_indexed, num_value_2, num_value_3_indexed)", "str_value_indexed", "num_value_2", "num_value_3_indexed");
    final Index newIndex2 = new Index("MySimpleRecord$(str_value_indexed, num_value_2, num_value_unique)", "str_value_indexed", "num_value_2", "num_value_unique");
    final RecordMetaDataHook addAllIndexesHook = composeHooks(addIndexHook(recordTypeName, origIndex), addIndexHook(recordTypeName, newIndex1), addIndexHook(recordTypeName, newIndex2));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, addAllIndexesHook);
        assertTrue(disableIndex(newIndex1));
        assertTrue(disableIndex(newIndex2));
        recordStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(800L).setStrValueIndexed("a_value").setNumValue2(962).setNumValue3Indexed(1806).build());
        commit(context);
    }
    // Mark the index as replaced by the two new indexes and expect the index to be gone once both new indexes are built
    RecordMetaDataHook withReplacementsHook = composeHooks(addIndexAndReplacements(recordTypeName, origIndex, newIndex1, newIndex2), bumpMetaDataVersionHook());
    final List<IndexEntry> origEntries;
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, withReplacementsHook);
        assertTrue(recordStore.isIndexReadable(origIndex));
        origEntries = scanIndex(origIndex);
        assertThat(origEntries, hasSize(1));
        assertEquals(Tuple.from(800L), origEntries.get(0).getPrimaryKey());
        assertEquals(Tuple.from("a_value", 962L, 800L), origEntries.get(0).getKey());
        buildIndex(newIndex1);
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, withReplacementsHook);
        assertTrue(recordStore.isIndexReadable(origIndex));
        assertEquals(origEntries, scanIndex(origIndex));
        disableIndex(newIndex1);
        buildIndex(newIndex2);
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, withReplacementsHook);
        assertTrue(recordStore.isIndexReadable(origIndex));
        assertEquals(origEntries, scanIndex(origIndex));
        buildIndex(newIndex1);
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, withReplacementsHook);
        assertTrue(recordStore.isIndexDisabled(origIndex));
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, composeHooks(addAllIndexesHook, bumpMetaDataVersionHook(), bumpMetaDataVersionHook()));
        assertTrue(uncheckedMarkIndexReadable(origIndex));
        assertThat(scanIndex(origIndex), empty());
    }
}

Example 48 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class OnlineIndexScrubberTest method testScrubberSimpleMissing.

@Test
void testScrubberSimpleMissing() throws ExecutionException, InterruptedException {
    final FDBStoreTimer timer = new FDBStoreTimer();
    final long numRecords = 50;
    long res;
    Index tgtIndex = new Index("tgt_index", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS);
    FDBRecordStoreTestBase.RecordMetaDataHook hook = myHook(tgtIndex);
    openSimpleMetaData();
    populateData(numRecords);
    openSimpleMetaData(hook);
    buildIndex(tgtIndex);
    try (OnlineIndexScrubber indexScrubber = OnlineIndexScrubber.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(tgtIndex).setSubspace(subspace).setScrubbingPolicy(OnlineIndexScrubber.ScrubbingPolicy.newBuilder().setLogWarningsLimit(Integer.MAX_VALUE).setAllowRepair(false).build()).setTimer(timer).build()) {
        res = indexScrubber.scrubMissingIndexEntries();
    }
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    assertEquals(0, timer.getCount(FDBStoreTimer.Counts.INDEX_SCRUBBER_MISSING_ENTRIES));
    assertEquals(0, res);
    // manually delete a few index entries
    openSimpleMetaData(hook);
    int missingCount = 0;
    try (FDBRecordContext context = openContext()) {
        List<IndexEntry> indexEntries = recordStore.scanIndex(tgtIndex, IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asList().get();
        for (int i = 3; i < numRecords; i *= 2) {
            final IndexEntry indexEntry = indexEntries.get(i);
            final Tuple valueKey = indexEntry.getKey();
            final byte[] keyBytes = recordStore.indexSubspace(tgtIndex).pack(valueKey);
            recordStore.getContext().ensureActive().clear(keyBytes);
            missingCount++;
        }
        context.commit();
    }
    // verify the missing entries are found and fixed
    timer.reset();
    try (OnlineIndexScrubber indexScrubber = OnlineIndexScrubber.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(tgtIndex).setSubspace(subspace).setScrubbingPolicy(OnlineIndexScrubber.ScrubbingPolicy.newBuilder().build()).setTimer(timer).build()) {
        res = indexScrubber.scrubMissingIndexEntries();
    }
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(missingCount, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    assertEquals(missingCount, timer.getCount(FDBStoreTimer.Counts.INDEX_SCRUBBER_MISSING_ENTRIES));
    assertEquals(missingCount, res);
    // now verify it's fixed
    timer.reset();
    try (OnlineIndexScrubber indexScrubber = OnlineIndexScrubber.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(tgtIndex).setSubspace(subspace).setScrubbingPolicy(OnlineIndexScrubber.ScrubbingPolicy.newBuilder().build()).setTimer(timer).build()) {
        res = indexScrubber.scrubMissingIndexEntries();
    }
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    assertEquals(0, timer.getCount(FDBStoreTimer.Counts.INDEX_SCRUBBER_MISSING_ENTRIES));
    assertEquals(0, res);
}

Example 49 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class FunctionKeyIndexTest method testCountIndex.

@Test
public void testCountIndex() throws Exception {
    Index index = new Index("group_index", new GroupingKeyExpression(function("substr", concat(field("str_value"), value(0), value(3))), 0), IndexTypes.COUNT);
    Records records = Records.create();
    for (int i = 0; i < 10; i++) {
        records.add(i, "ab" + Character.toString((char) ('c' + (i % 3))) + "_" + i);
    }
    saveRecords(records, index);
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, index);
        List<IndexEntry> keyValues = getIndexKeyValues(recordStore, index, IndexScanType.BY_GROUP);
        assertEquals(Tuple.from("abc"), keyValues.get(0).getKey());
        assertEquals(Tuple.from(4), keyValues.get(0).getValue());
        assertEquals(Tuple.from("abd"), keyValues.get(1).getKey());
        assertEquals(Tuple.from(3), keyValues.get(1).getValue());
        assertEquals(Tuple.from("abe"), keyValues.get(2).getKey());
        assertEquals(Tuple.from(3), keyValues.get(2).getValue());
    }
}

Example 50 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class TextIndexTest method scanWithZeroScanRecordLimit.

private void scanWithZeroScanRecordLimit(@Nonnull Index index, @Nonnull String token, boolean reverse) throws Exception {
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        ScanProperties scanProperties = ExecuteProperties.newBuilder().setScannedRecordsLimit(0).build().asScanProperties(reverse);
        RecordCursor<IndexEntry> cursor = recordStore.scanIndex(index, BY_TEXT_TOKEN, TupleRange.allOf(Tuple.from(token)), null, scanProperties);
        RecordCursorResult<IndexEntry> result = cursor.getNext();
        if (!result.hasNext()) {
            assertEquals(SOURCE_EXHAUSTED, result.getNoNextReason());
            return;
        }
        result = cursor.getNext();
        assertThat(result.hasNext(), is(false));
        assertEquals(SCAN_LIMIT_REACHED, result.getNoNextReason());
    }
}