Examples with IndexEntry com.apple.foundationdb.record.IndexEntry used on opensource projects

Example 1 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class IndexingScrubDangling method deleteIndexIfDangling.

@Nullable
private CompletableFuture<FDBStoredRecord<Message>> deleteIndexIfDangling(FDBRecordStore store, final RecordCursorResult<FDBIndexedRecord<Message>> cursorResult) {
    // This will always return null (!) - but sometimes will delete a dangling index
    FDBIndexedRecord<Message> indexResult = cursorResult.get();
    if (!indexResult.hasStoredRecord()) {
        // Here: Oh, No! this index is dangling!
        danglingCount.incrementAndGet();
        final FDBStoreTimer timer = getRunner().getTimer();
        timerIncrement(timer, FDBStoreTimer.Counts.INDEX_SCRUBBER_DANGLING_ENTRIES);
        final IndexEntry indexEntry = indexResult.getIndexEntry();
        final Tuple valueKey = indexEntry.getKey();
        final byte[] keyBytes = store.indexSubspace(common.getIndex()).pack(valueKey);
        if (LOGGER.isWarnEnabled() && logWarningCounter > 0) {
            logWarningCounter--;
            LOGGER.warn(KeyValueLogMessage.build("Scrubber: dangling index entry", LogMessageKeys.KEY, valueKey.toString()).addKeysAndValues(common.indexLogMessageKeyValues()).toString());
        }
        if (scrubbingPolicy.allowRepair()) {
            // remove this index entry
            // Note that there no record can be added to the conflict list, so we'll add the index entry itself.
            store.addRecordReadConflict(indexEntry.getPrimaryKey());
            store.getContext().ensureActive().clear(keyBytes);
        }
    }
    return CompletableFuture.completedFuture(null);
}

Example 2 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class IndexingScrubMissing method getRecordIfMissingIndex.

@Nullable
private CompletableFuture<FDBStoredRecord<Message>> getRecordIfMissingIndex(FDBRecordStore store, final RecordCursorResult<FDBStoredRecord<Message>> currResult) {
    final FDBStoredRecord<Message> rec = currResult.get();
    // return true if an index is missing and updated
    if (!common.getAllRecordTypes().contains(rec.getRecordType())) {
        return CompletableFuture.completedFuture(null);
    }
    final Index index = common.getIndex();
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    List<IndexEntry> indexEntryNoPKs = maintainer.filteredIndexEntries(rec);
    if (indexEntryNoPKs == null) {
        return CompletableFuture.completedFuture(null);
    }
    return AsyncUtil.getAll(indexEntryNoPKs.stream().map(entry -> {
        // should I convert it to a single nested statement?
        final IndexEntry indexEntry = new IndexEntry(index, FDBRecordStoreBase.indexEntryKey(index, entry.getKey(), rec.getPrimaryKey()), entry.getValue());
        final Tuple valueKey = indexEntry.getKey();
        final byte[] keyBytes = maintainer.getIndexSubspace().pack(valueKey);
        return maintainer.state.transaction.get(keyBytes).thenApply(indexVal -> indexVal == null ? valueKey : null);
    }).collect(Collectors.toList())).thenApply(list -> {
        List<Tuple> missingIndexesKeys = list.stream().filter(Objects::nonNull).collect(Collectors.toList());
        if (missingIndexesKeys.isEmpty()) {
            return null;
        }
        // (Maybe) report an error and (maybe) return this record to be index
        if (LOGGER.isWarnEnabled() && logWarningCounter > 0) {
            logWarningCounter--;
            LOGGER.warn(KeyValueLogMessage.build("Scrubber: missing index entry", LogMessageKeys.KEY, rec.getPrimaryKey().toString(), LogMessageKeys.INDEX_KEY, missingIndexesKeys.toString()).addKeysAndValues(common.indexLogMessageKeyValues()).toString());
        }
        missingCount.incrementAndGet();
        final FDBStoreTimer timer = getRunner().getTimer();
        timerIncrement(timer, FDBStoreTimer.Counts.INDEX_SCRUBBER_MISSING_ENTRIES);
        if (scrubbingPolicy.allowRepair()) {
            // record to be indexed
            return rec;
        }
        // report only mode
        return null;
    });
}

Example 3 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class StandardIndexMaintainer method filteredIndexEntries.

/**
 * Filter out index keys according to {@link IndexMaintenanceFilter}.
 * Keys that do not pass the filter will not be stored / removed from the index.
 * @param <M> the message type of the record
 * @param savedRecord record for key evaluation
 * @return filtered list of index keys for the given record
 */
@Override
@Nullable
public <M extends Message> List<IndexEntry> filteredIndexEntries(@Nullable final FDBIndexableRecord<M> savedRecord) {
    if (savedRecord == null) {
        return null;
    }
    final Message record = savedRecord.getRecord();
    long startTime = System.nanoTime();
    boolean filterIndexKeys = false;
    switch(state.filter.maintainIndex(state.index, record)) {
        case NONE:
            if (state.store.getTimer() != null) {
                state.store.getTimer().recordSinceNanoTime(FDBStoreTimer.Events.SKIP_INDEX_RECORD, startTime);
            }
            return null;
        case SOME:
            filterIndexKeys = true;
            break;
        case ALL:
        default:
            break;
    }
    List<IndexEntry> indexEntries = evaluateIndex(savedRecord);
    if (!filterIndexKeys) {
        return indexEntries;
    }
    int i = 0;
    while (i < indexEntries.size()) {
        if (state.filter.maintainIndexValue(state.index, record, indexEntries.get(i))) {
            i++;
        } else {
            indexEntries = makeMutable(indexEntries);
            indexEntries.remove(i);
            long endTime = System.nanoTime();
            if (state.store.getTimer() != null) {
                state.store.getTimer().record(FDBStoreTimer.Events.SKIP_INDEX_ENTRY, endTime - startTime);
            }
            startTime = endTime;
        }
    }
    return indexEntries;
}

Example 4 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class StandardIndexMaintainer method evaluateIndex.

/**
 * Apply the key and value expressions to a <code>record</code>.
 * @param <M> the message type of the record
 * @param record the record from which the index will extract its key and value
 * @return a list of index keys and values
 */
@Override
public <M extends Message> List<IndexEntry> evaluateIndex(@Nonnull FDBRecord<M> record) {
    final KeyExpression rootExpression = state.index.getRootExpression();
    final List<Key.Evaluated> indexKeys = rootExpression.evaluate(record);
    // so we have to tease them apart.
    if (rootExpression instanceof KeyWithValueExpression) {
        final KeyWithValueExpression keyWithValueExpression = (KeyWithValueExpression) rootExpression;
        return indexKeys.stream().map(key -> new IndexEntry(state.index, keyWithValueExpression.getKey(key), keyWithValueExpression.getValue(key))).collect(Collectors.toList());
    }
    return indexKeys.stream().map(key -> new IndexEntry(state.index, key)).collect(Collectors.toList());
}

Example 5 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class StandardIndexMaintainer method checkUniqueness.

protected <M extends Message> void checkUniqueness(@Nonnull FDBIndexableRecord<M> savedRecord, @Nonnull IndexEntry indexEntry) {
    Tuple valueKey = indexEntry.getKey();
    AsyncIterable<KeyValue> kvs = state.transaction.getRange(state.indexSubspace.range(valueKey));
    Tuple primaryKey = savedRecord.getPrimaryKey();
    final CompletableFuture<Void> checker = state.store.getContext().instrument(FDBStoreTimer.Events.CHECK_INDEX_UNIQUENESS, AsyncUtil.forEach(kvs, kv -> {
        Tuple existingEntry = unpackKey(getIndexSubspace(), kv);
        Tuple existingKey = state.index.getEntryPrimaryKey(existingEntry);
        if (!TupleHelpers.equals(primaryKey, existingKey)) {
            if (state.store.isIndexWriteOnly(state.index)) {
                addUniquenessViolation(valueKey, primaryKey, existingKey);
                addUniquenessViolation(valueKey, existingKey, primaryKey);
            } else {
                throw new RecordIndexUniquenessViolation(state.index, indexEntry, primaryKey, existingKey);
            }
        }
    }, getExecutor()));
    // Add a pre-commit check to prevent accidentally committing and getting into an invalid state.
    state.store.addIndexUniquenessCommitCheck(state.index, checker);
}