Examples with IndexState com.apple.foundationdb.record.IndexState used on opensource projects

Example 6 with IndexState

use of com.apple.foundationdb.record.IndexState in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method noMaintenanceFilteredIndexOnCheckVersion.

@Test
public void noMaintenanceFilteredIndexOnCheckVersion() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openAnyRecordStore(TestRecordsIndexFilteringProto.getDescriptor(), context);
        TestRecordsIndexFilteringProto.MyBasicRecord recordA = TestRecordsIndexFilteringProto.MyBasicRecord.newBuilder().setRecNo(1001).setNumValue2(101).build();
        recordStore.saveRecord(recordA);
        context.commit();
        Collection<StoreTimer.Event> events = context.getTimer().getEvents();
        assertFalse(events.contains(FDBStoreTimer.Events.SAVE_INDEX_ENTRY));
        assertFalse(events.contains(FDBStoreTimer.Events.SKIP_INDEX_RECORD));
    }
    // add index
    RecordMetaDataHook hook = metaDataBuilder -> {
        metaDataBuilder.setVersion(metaDataBuilder.getVersion() + 1);
        metaDataBuilder.addIndex("MyBasicRecord", "value2$filtered", field("num_value_2"));
    };
    try (FDBRecordContext context = openContext()) {
        RecordMetaDataBuilder metaData = RecordMetaData.newBuilder().setRecords(TestRecordsIndexFilteringProto.getDescriptor());
        if (hook != null) {
            hook.apply(metaData);
        }
        IndexMaintenanceFilter noneFilter = (i, r) -> IndexMaintenanceFilter.IndexValues.NONE;
        FDBRecordStore fdbRecordStore = FDBRecordStore.newBuilder().setContext(context).setMetaDataProvider(metaData).setKeySpacePath(path).setIndexMaintenanceFilter(noneFilter).uncheckedOpen();
        context.getTimer().reset();
        FDBRecordStoreBase.UserVersionChecker userVersionChecker = new FDBRecordStoreBase.UserVersionChecker() {

            @Override
            public CompletableFuture<Integer> checkUserVersion(int oldUserVersion, int oldMetaDataVersion, RecordMetaDataProvider metaData) {
                return CompletableFuture.completedFuture(Integer.valueOf(1));
            }

            @Override
            public IndexState needRebuildIndex(Index index, long recordCount, boolean indexOnNewRecordTypes) {
                return IndexState.READABLE;
            }
        };
        fdbRecordStore.checkVersion(userVersionChecker, FDBRecordStoreBase.StoreExistenceCheck.NONE).get();
        Collection<StoreTimer.Event> events = context.getTimer().getEvents();
        assertFalse(events.contains(FDBStoreTimer.Events.SAVE_INDEX_ENTRY));
        assertTrue(events.contains(FDBStoreTimer.Events.SKIP_INDEX_RECORD));
    }
}

Example 7 with IndexState

use of com.apple.foundationdb.record.IndexState in project fdb-record-layer by FoundationDB.

the class OnlineIndexerBuildIndexTest method singleRebuild.

void singleRebuild(@Nonnull List<TestRecords1Proto.MySimpleRecord> records, @Nullable List<TestRecords1Proto.MySimpleRecord> recordsWhileBuilding, int agents, boolean overlap, boolean splitLongRecords, @Nonnull Index index, @Nonnull Runnable beforeBuild, @Nonnull Runnable afterBuild, @Nonnull Runnable afterReadable) {
    LOGGER.info(KeyValueLogMessage.of("beginning rebuild test", TestLogMessageKeys.RECORDS, records.size(), LogMessageKeys.RECORDS_WHILE_BUILDING, recordsWhileBuilding == null ? 0 : recordsWhileBuilding.size(), TestLogMessageKeys.AGENTS, agents, TestLogMessageKeys.OVERLAP, overlap, TestLogMessageKeys.SPLIT_LONG_RECORDS, splitLongRecords, TestLogMessageKeys.INDEX, index));
    final FDBStoreTimer timer = new FDBStoreTimer();
    final FDBRecordStoreTestBase.RecordMetaDataHook onlySplitHook = metaDataBuilder -> {
        if (splitLongRecords) {
            metaDataBuilder.setSplitLongRecords(true);
            metaDataBuilder.removeIndex("MySimpleRecord$str_value_indexed");
        }
    };
    final FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> {
        onlySplitHook.apply(metaDataBuilder);
        metaDataBuilder.addIndex("MySimpleRecord", index);
    };
    LOGGER.info(KeyValueLogMessage.of("inserting elements prior to test", TestLogMessageKeys.RECORDS, records.size()));
    openSimpleMetaData(onlySplitHook);
    try (FDBRecordContext context = openContext()) {
        for (TestRecords1Proto.MySimpleRecord record : records) {
            // Check presence first to avoid overwriting version information of previously added records.
            Tuple primaryKey = Tuple.from(record.getRecNo());
            if (recordStore.loadRecord(primaryKey) == null) {
                recordStore.saveRecord(record);
            }
        }
        context.commit();
    }
    LOGGER.info(KeyValueLogMessage.of("running before build for test"));
    beforeBuild.run();
    openSimpleMetaData(hook);
    LOGGER.info(KeyValueLogMessage.of("adding index", TestLogMessageKeys.INDEX, index));
    openSimpleMetaData(hook);
    final boolean isAlwaysReadable;
    try (FDBRecordContext context = openContext()) {
        // care of by OnlineIndexer.
        if (!safeBuild) {
            LOGGER.info(KeyValueLogMessage.of("marking write-only", TestLogMessageKeys.INDEX, index));
            recordStore.clearAndMarkIndexWriteOnly(index).join();
        }
        isAlwaysReadable = safeBuild && recordStore.isIndexReadable(index);
        context.commit();
    }
    LOGGER.info(KeyValueLogMessage.of("creating online index builder", TestLogMessageKeys.INDEX, index, TestLogMessageKeys.RECORD_TYPES, metaData.recordTypesForIndex(index), LogMessageKeys.SUBSPACE, ByteArrayUtil2.loggable(subspace.pack()), LogMessageKeys.LIMIT, 20, TestLogMessageKeys.RECORDS_PER_SECOND, OnlineIndexer.DEFAULT_RECORDS_PER_SECOND * 100));
    final OnlineIndexer.Builder builder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).setConfigLoader(old -> {
        OnlineIndexer.Config.Builder conf = OnlineIndexer.Config.newBuilder().setMaxLimit(20).setMaxRetries(Integer.MAX_VALUE).setRecordsPerSecond(OnlineIndexer.DEFAULT_RECORDS_PER_SECOND * 100);
        if (ThreadLocalRandom.current().nextBoolean()) {
            // randomly enable the progress logging to ensure that it doesn't throw exceptions,
            // or otherwise disrupt the build.
            LOGGER.info("Setting progress log interval");
            conf.setProgressLogIntervalMillis(0);
        }
        return conf.build();
    }).setTimer(timer);
    if (ThreadLocalRandom.current().nextBoolean()) {
        LOGGER.info("Setting priority to DEFAULT");
        builder.setPriority(FDBTransactionPriority.DEFAULT);
    }
    if (fdb.isTrackLastSeenVersion()) {
        LOGGER.info("Setting weak read semantics");
        builder.setWeakReadSemantics(new FDBDatabase.WeakReadSemantics(0L, Long.MAX_VALUE, true));
    }
    if (!safeBuild) {
        builder.setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setIfDisabled(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR).setIfMismatchPrevious(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR));
        builder.setUseSynchronizedSession(false);
    }
    try (OnlineIndexer indexBuilder = builder.build()) {
        CompletableFuture<Void> buildFuture;
        LOGGER.info(KeyValueLogMessage.of("building index", TestLogMessageKeys.INDEX, index, TestLogMessageKeys.AGENT, agents, LogMessageKeys.RECORDS_WHILE_BUILDING, recordsWhileBuilding == null ? 0 : recordsWhileBuilding.size(), TestLogMessageKeys.OVERLAP, overlap));
        if (agents == 1) {
            buildFuture = indexBuilder.buildIndexAsync(false);
        } else {
            if (overlap) {
                CompletableFuture<?>[] futures = new CompletableFuture<?>[agents];
                for (int i = 0; i < agents; i++) {
                    final int agent = i;
                    futures[i] = safeBuild ? indexBuilder.buildIndexAsync(false).exceptionally(exception -> {
                        // because the other one is already working on building the index.
                        if (exception.getCause() instanceof SynchronizedSessionLockedException) {
                            LOGGER.info(KeyValueLogMessage.of("Detected another worker processing this index", TestLogMessageKeys.INDEX, index, TestLogMessageKeys.AGENT, agent), exception);
                            return null;
                        } else {
                            throw new CompletionException(exception);
                        }
                    }) : indexBuilder.buildIndexAsync(false);
                }
                buildFuture = CompletableFuture.allOf(futures);
            } else {
                // Safe builds do not support building ranges yet.
                assumeFalse(safeBuild);
                buildFuture = indexBuilder.buildEndpoints().thenCompose(tupleRange -> {
                    if (tupleRange != null) {
                        long start = tupleRange.getLow().getLong(0);
                        long end = tupleRange.getHigh().getLong(0);
                        CompletableFuture<?>[] futures = new CompletableFuture<?>[agents];
                        for (int i = 0; i < agents; i++) {
                            long itrStart = start + (end - start) / agents * i;
                            long itrEnd = (i == agents - 1) ? end : start + (end - start) / agents * (i + 1);
                            LOGGER.info(KeyValueLogMessage.of("building range", TestLogMessageKeys.INDEX, index, TestLogMessageKeys.AGENT, i, TestLogMessageKeys.BEGIN, itrStart, TestLogMessageKeys.END, itrEnd));
                            futures[i] = indexBuilder.buildRange(Key.Evaluated.scalar(itrStart), Key.Evaluated.scalar(itrEnd));
                        }
                        return CompletableFuture.allOf(futures);
                    } else {
                        return AsyncUtil.DONE;
                    }
                });
            }
        }
        if (safeBuild) {
            buildFuture = MoreAsyncUtil.composeWhenComplete(buildFuture, (result, ex) -> indexBuilder.checkAnyOngoingOnlineIndexBuildsAsync().thenAccept(Assertions::assertFalse), fdb::mapAsyncToSyncException);
        }
        if (recordsWhileBuilding != null && recordsWhileBuilding.size() > 0) {
            int i = 0;
            while (i < recordsWhileBuilding.size()) {
                List<TestRecords1Proto.MySimpleRecord> thisBatch = recordsWhileBuilding.subList(i, Math.min(i + 30, recordsWhileBuilding.size()));
                fdb.run(context -> {
                    FDBRecordStore store = recordStore.asBuilder().setContext(context).build();
                    thisBatch.forEach(store::saveRecord);
                    return null;
                });
                i += 30;
            }
        }
        buildFuture.join();
        // if a record is added to a range that has already been built, it will not be counted, otherwise,
        // it will.
        long additionalScans = 0;
        if (recordsWhileBuilding != null && recordsWhileBuilding.size() > 0) {
            additionalScans += (long) recordsWhileBuilding.size();
        }
        try (FDBRecordContext context = openContext()) {
            IndexBuildState indexBuildState = context.asyncToSync(FDBStoreTimer.Waits.WAIT_GET_INDEX_BUILD_STATE, IndexBuildState.loadIndexBuildStateAsync(recordStore, index));
            IndexState indexState = indexBuildState.getIndexState();
            if (isAlwaysReadable) {
                assertEquals(IndexState.READABLE, indexState);
            } else {
                assertEquals(IndexState.WRITE_ONLY, indexState);
                assertEquals(indexBuilder.getTotalRecordsScanned(), indexBuildState.getRecordsScanned());
                // Count index is not defined so we cannot determine the records in total from it.
                assertNull(indexBuildState.getRecordsInTotal());
            }
        }
        assertThat(indexBuilder.getTotalRecordsScanned(), allOf(greaterThanOrEqualTo((long) records.size()), lessThanOrEqualTo((long) records.size() + additionalScans)));
    }
    KeyValueLogMessage msg = KeyValueLogMessage.build("building index - completed", TestLogMessageKeys.INDEX, index);
    msg.addKeysAndValues(timer.getKeysAndValues());
    LOGGER.info(msg.toString());
    LOGGER.info(KeyValueLogMessage.of("running post build checks", TestLogMessageKeys.INDEX, index));
    // uses the index in quereis since the index is readable.
    if (!isAlwaysReadable) {
        afterBuild.run();
    }
    LOGGER.info(KeyValueLogMessage.of("verifying range set emptiness", TestLogMessageKeys.INDEX, index));
    try (FDBRecordContext context = openContext()) {
        RangeSet rangeSet = new RangeSet(recordStore.indexRangeSubspace(metaData.getIndex(index.getName())));
        System.out.println("Range set for " + records.size() + " records:\n" + rangeSet.rep(context.ensureActive()).join());
        if (!isAlwaysReadable) {
            assertEquals(Collections.emptyList(), rangeSet.missingRanges(context.ensureActive()).asList().join());
        }
        context.commit();
    }
    LOGGER.info(KeyValueLogMessage.of("marking index readable", TestLogMessageKeys.INDEX, index));
    try (FDBRecordContext context = openContext()) {
        boolean updated = recordStore.markIndexReadable(index).join();
        if (isAlwaysReadable) {
            assertFalse(updated);
        } else {
            assertTrue(updated);
        }
        context.commit();
    }
    afterReadable.run();
    LOGGER.info(KeyValueLogMessage.of("ending rebuild test", TestLogMessageKeys.RECORDS, records.size(), LogMessageKeys.RECORDS_WHILE_BUILDING, recordsWhileBuilding == null ? 0 : recordsWhileBuilding.size(), TestLogMessageKeys.AGENTS, agents, TestLogMessageKeys.OVERLAP, overlap, TestLogMessageKeys.SPLIT_LONG_RECORDS, splitLongRecords, TestLogMessageKeys.INDEX, index));
}

Example 8 with IndexState

use of com.apple.foundationdb.record.IndexState in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method vacuumReadableIndexesBuildData.

public void vacuumReadableIndexesBuildData() {
    Transaction tr = ensureContextActive();
    // also adds state to read conflicts
    Map<Index, IndexState> indexStates = getAllIndexStates();
    for (Map.Entry<Index, IndexState> entry : indexStates.entrySet()) {
        if (entry.getValue().equals(IndexState.READABLE)) {
            clearReadableIndexBuildData(tr, entry.getKey());
        }
    }
}

Example 9 with IndexState

use of com.apple.foundationdb.record.IndexState in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method rebuildIndexes.

@Nonnull
protected CompletableFuture<Void> rebuildIndexes(@Nonnull Map<Index, List<RecordType>> indexes, @Nonnull Map<Index, CompletableFuture<IndexState>> newStates, @Nonnull List<CompletableFuture<Void>> work, @Nonnull RebuildIndexReason reason, @Nullable Integer oldMetaDataVersion) {
    Iterator<Map.Entry<Index, List<RecordType>>> indexIter = indexes.entrySet().iterator();
    return AsyncUtil.whileTrue(() -> {
        Iterator<CompletableFuture<Void>> workIter = work.iterator();
        while (workIter.hasNext()) {
            CompletableFuture<Void> workItem = workIter.next();
            if (workItem.isDone()) {
                // Just for error handling.
                context.asyncToSync(FDBStoreTimer.Waits.WAIT_ERROR_CHECK, workItem);
                workIter.remove();
            }
        }
        while (work.size() < MAX_PARALLEL_INDEX_REBUILD) {
            if (indexIter.hasNext()) {
                Map.Entry<Index, List<RecordType>> indexItem = indexIter.next();
                Index index = indexItem.getKey();
                List<RecordType> recordTypes = indexItem.getValue();
                final CompletableFuture<Void> rebuildOrMarkIndexSafely = MoreAsyncUtil.handleOnException(() -> newStates.getOrDefault(index, READY_READABLE).thenCompose(indexState -> rebuildOrMarkIndex(index, indexState, recordTypes, reason, oldMetaDataVersion)), exception -> {
                    // If there is any issue, simply mark the index as disabled without blocking checkVersion
                    logExceptionAsWarn(KeyValueLogMessage.build("unable to build index", LogMessageKeys.INDEX_NAME, index.getName()), exception);
                    return markIndexDisabled(index).thenApply(b -> null);
                });
                work.add(rebuildOrMarkIndexSafely);
            } else {
                break;
            }
        }
        if (work.isEmpty()) {
            return AsyncUtil.READY_FALSE;
        }
        return AsyncUtil.whenAny(work).thenApply(v -> true);
    }, getExecutor());
}

Example 10 with IndexState

use of com.apple.foundationdb.record.IndexState in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method getStatesForRebuildIndexes.

@Nonnull
protected Map<Index, CompletableFuture<IndexState>> getStatesForRebuildIndexes(@Nullable UserVersionChecker userVersionChecker, @Nonnull Map<Index, List<RecordType>> indexes, @Nonnull Supplier<CompletableFuture<Long>> lazyRecordCount, @Nonnull Supplier<CompletableFuture<Long>> lazyRecordsSize, boolean newStore, int oldMetaDataVersion, int oldFormatVersion) {
    Map<Index, CompletableFuture<IndexState>> newStates = new HashMap<>();
    for (Map.Entry<Index, List<RecordType>> entry : indexes.entrySet()) {
        Index index = entry.getKey();
        List<RecordType> recordTypes = entry.getValue();
        boolean indexOnNewRecordTypes = areAllRecordTypesSince(recordTypes, oldMetaDataVersion);
        CompletableFuture<IndexState> stateFuture = userVersionChecker == null ? lazyRecordCount.get().thenApply(recordCount -> FDBRecordStore.disabledIfTooManyRecordsForRebuild(recordCount, indexOnNewRecordTypes)) : userVersionChecker.needRebuildIndex(index, lazyRecordCount, lazyRecordsSize, indexOnNewRecordTypes);
        if (index.getType().equals(IndexTypes.VERSION) && !newStore && oldFormatVersion < SAVE_VERSION_WITH_RECORD_FORMAT_VERSION && !useOldVersionFormat()) {
            stateFuture = stateFuture.thenApply(state -> {
                if (IndexState.READABLE.equals(state)) {
                    // build and some versions won't be indexed correctly.
                    return IndexState.DISABLED;
                }
                return state;
            });
        }
        newStates.put(index, stateFuture);
    }
    return newStates;
}