Examples with Index com.apple.foundationdb.record.metadata.Index used on opensource projects

Example 21 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class IndexingByRecords method buildRangeOnly.

@Nonnull
private CompletableFuture<Tuple> buildRangeOnly(@Nonnull FDBRecordStore store, @Nonnull TupleRange range, boolean respectLimit, @Nullable AtomicLong recordsScanned) {
    validateSameMetadataOrThrow(store);
    Index index = common.getIndex();
    int limit = getLimit();
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    final boolean isIdempotent = maintainer.isIdempotent();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(isIdempotent ? IsolationLevel.SNAPSHOT : IsolationLevel.SERIALIZABLE);
    if (respectLimit) {
        // +1 allows continuation item
        executeProperties.setReturnedRowLimit(limit + 1);
    }
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    final RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(range, null, scanProperties);
    final AtomicBoolean hasMore = new AtomicBoolean(true);
    // Note: This runs all of the updates serially in order to avoid invoking a race condition
    // in the rank code that was causing incorrect results. If everything were thread safe,
    // a larger pipeline size would be possible.
    final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
    return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : range.getHigh());
}

Example 22 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class IndexingBase method setScrubberTypeOrThrow.

@Nonnull
private CompletableFuture<Void> setScrubberTypeOrThrow(FDBRecordStore store) {
    // HERE: The index must be readable, checked by the caller
    // if scrubber had already run and still have missing ranges, do nothing
    // else: clear ranges and overwrite type-stamp
    Transaction tr = store.getContext().ensureActive();
    IndexBuildProto.IndexBuildIndexingStamp indexingTypeStamp = getIndexingTypeStamp(store);
    validateOrThrowEx(indexingTypeStamp.getMethod().equals(IndexBuildProto.IndexBuildIndexingStamp.Method.SCRUB_REPAIR), "Not a scrubber type-stamp");
    // Note: the scrubbers do not support multi target (yet)
    final Index index = common.getIndex();
    final Subspace indexesRangeSubspace = indexScrubIndexRangeSubspace(store, index);
    final Subspace recordsRangeSubspace = indexScrubRecordsRangeSubspace(store, index);
    RangeSet indexRangeSet = new RangeSet(indexesRangeSubspace);
    RangeSet recordsRangeSet = new RangeSet(recordsRangeSubspace);
    AsyncIterator<Range> indexRanges = indexRangeSet.missingRanges(tr).iterator();
    AsyncIterator<Range> recordsRanges = recordsRangeSet.missingRanges(tr).iterator();
    return indexRanges.onHasNext().thenCompose(hasNextIndex -> {
        if (Boolean.FALSE.equals(hasNextIndex)) {
            // erase the 'ranges' data to allow a fresh index re-scrubbing.
            if (LOGGER.isInfoEnabled()) {
                LOGGER.info(KeyValueLogMessage.build("Reset scrubber's index range").addKeysAndValues(common.indexLogMessageKeyValues()).toString());
            }
            tr.clear(indexesRangeSubspace.range());
        }
        return recordsRanges.onHasNext().thenAccept(hasNextRecord -> {
            if (Boolean.FALSE.equals(hasNextRecord)) {
                // erase the 'ranges' data to allow a fresh records re-scrubbing.
                if (LOGGER.isInfoEnabled()) {
                    LOGGER.info(KeyValueLogMessage.build("Reset scrubber's records range").addKeysAndValues(common.indexLogMessageKeyValues()).toString());
                }
                tr.clear(recordsRangeSubspace.range());
            }
        });
    });
}

Example 23 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class RecordMetaData method toProto.

/**
 * Serializes the record meta-data to a <code>MetaData</code> proto message. This operates like
 * {@link #toProto()} except that any dependency in the excluded list is not included in the
 * serialized proto message. If the list is set to {@code null}, then all dependencies will be serialized
 * to the proto message including those that are execluded by default.
 *
 * @param excludedDependencies a list of dependencies not to include in the serialized proto
 * @return the serialized <code>MetaData</code> proto message
 * @throws KeyExpression.SerializationException on any serialization failures
 * @throws MetaDataException if this {@code RecordMetaData} was initialized with a
 *      {@linkplain RecordMetaDataBuilder#setLocalFileDescriptor(Descriptors.FileDescriptor) local file descriptor}
 * @see #toProto()
 */
@Nonnull
@SuppressWarnings("deprecation")
public RecordMetaDataProto.MetaData toProto(@Nullable Descriptors.FileDescriptor[] excludedDependencies) throws KeyExpression.SerializationException {
    if (usesLocalRecordsDescriptor) {
        throw new MetaDataException("cannot serialize meta-data with a local records descriptor to proto");
    }
    RecordMetaDataProto.MetaData.Builder builder = RecordMetaDataProto.MetaData.newBuilder();
    // Set the root records.
    builder.setRecords(recordsDescriptor.toProto());
    // Convert the exclusion list to a map
    Map<String, Descriptors.FileDescriptor> excludeMap = null;
    if (excludedDependencies != null) {
        excludeMap = new HashMap<>(excludedDependencies.length);
        for (Descriptors.FileDescriptor dependency : excludedDependencies) {
            excludeMap.put(dependency.getName(), dependency);
        }
    }
    // Add in the rest of dependencies.
    Map<String, Descriptors.FileDescriptor> allDependencies = new TreeMap<>();
    getDependencies(recordsDescriptor, allDependencies, excludeMap);
    for (Descriptors.FileDescriptor dependency : allDependencies.values()) {
        builder.addDependencies(dependency.toProto());
    }
    // Create builders for each index so that we can then add associated record types (etc.).
    Map<String, RecordMetaDataProto.Index.Builder> indexBuilders = new TreeMap<>();
    for (Map.Entry<String, Index> entry : indexes.entrySet()) {
        indexBuilders.put(entry.getKey(), entry.getValue().toProto().toBuilder());
    }
    for (RecordType recordType : getRecordTypes().values()) {
        // Add this record type to each appropriate index.
        for (Index index : recordType.getIndexes()) {
            indexBuilders.get(index.getName()).addRecordType(recordType.getName());
        }
        for (Index index : recordType.getMultiTypeIndexes()) {
            indexBuilders.get(index.getName()).addRecordType(recordType.getName());
        }
        RecordMetaDataProto.RecordType.Builder typeBuilder = builder.addRecordTypesBuilder().setName(recordType.getName()).setPrimaryKey(recordType.getPrimaryKey().toKeyExpression());
        if (recordType.getSinceVersion() != null) {
            typeBuilder.setSinceVersion(recordType.getSinceVersion());
        }
        if (recordType.hasExplicitRecordTypeKey()) {
            typeBuilder.setExplicitKey(LiteralKeyExpression.toProtoValue(recordType.getExplicitRecordTypeKey()));
        }
    }
    indexBuilders.values().forEach(builder::addIndexes);
    // Add in the former indexes.
    for (FormerIndex formerIndex : getFormerIndexes()) {
        builder.addFormerIndexes(formerIndex.toProto());
    }
    // Add in the final options.
    builder.setSplitLongRecords(splitLongRecords);
    builder.setStoreRecordVersions(storeRecordVersions);
    builder.setVersion(version);
    if (usesSubspaceKeyCounter()) {
        builder.setSubspaceKeyCounter(subspaceKeyCounter);
        builder.setUsesSubspaceKeyCounter(true);
    }
    if (recordCountKey != null) {
        builder.setRecordCountKey(recordCountKey.toKeyExpression());
    }
    for (SyntheticRecordType<?> syntheticRecordType : syntheticRecordTypes.values()) {
        if (syntheticRecordType instanceof JoinedRecordType) {
            builder.addJoinedRecordTypes(((JoinedRecordType) syntheticRecordType).toProto());
        }
    }
    return builder.build();
}

Example 24 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class RecordMetaDataBuilder method removeIndex.

public void removeIndex(@Nonnull String name) {
    Index index = indexes.remove(name);
    if (index == null) {
        throw new MetaDataException("No index named " + name + " defined");
    }
    for (RecordTypeBuilder recordType : recordTypes.values()) {
        recordType.getIndexes().remove(index);
        recordType.getMultiTypeIndexes().remove(index);
    }
    universalIndexes.remove(name);
    formerIndexes.add(new FormerIndex(index.getSubspaceKey(), index.getAddedVersion(), ++version, name));
}

Example 25 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class RecordMetaDataBuilder method loadProtoExceptRecords.

@SuppressWarnings("deprecation")
private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDataProto) {
    for (RecordMetaDataProto.Index indexProto : metaDataProto.getIndexesList()) {
        List<RecordTypeBuilder> recordTypeBuilders = new ArrayList<>(indexProto.getRecordTypeCount());
        for (String recordTypeName : indexProto.getRecordTypeList()) {
            recordTypeBuilders.add(getRecordType(recordTypeName));
        }
        try {
            addMultiTypeIndex(recordTypeBuilders, new Index(indexProto));
        } catch (KeyExpression.DeserializationException e) {
            throw new MetaDataProtoDeserializationException(e);
        }
    }
    for (RecordMetaDataProto.RecordType typeProto : metaDataProto.getRecordTypesList()) {
        RecordTypeBuilder typeBuilder = getRecordType(typeProto.getName());
        if (typeProto.hasPrimaryKey()) {
            try {
                typeBuilder.setPrimaryKey(KeyExpression.fromProto(typeProto.getPrimaryKey()));
            } catch (KeyExpression.DeserializationException e) {
                throw new MetaDataProtoDeserializationException(e);
            }
        }
        if (typeProto.hasSinceVersion()) {
            typeBuilder.setSinceVersion(typeProto.getSinceVersion());
        }
        if (typeProto.hasExplicitKey()) {
            typeBuilder.setRecordTypeKey(LiteralKeyExpression.fromProtoValue(typeProto.getExplicitKey()));
        }
    }
    if (metaDataProto.hasSplitLongRecords()) {
        splitLongRecords = metaDataProto.getSplitLongRecords();
    }
    if (metaDataProto.hasStoreRecordVersions()) {
        storeRecordVersions = metaDataProto.getStoreRecordVersions();
    }
    for (RecordMetaDataProto.FormerIndex formerIndex : metaDataProto.getFormerIndexesList()) {
        formerIndexes.add(new FormerIndex(formerIndex));
    }
    if (metaDataProto.hasRecordCountKey()) {
        try {
            recordCountKey = KeyExpression.fromProto(metaDataProto.getRecordCountKey());
        } catch (KeyExpression.DeserializationException e) {
            throw new MetaDataProtoDeserializationException(e);
        }
    }
    if (metaDataProto.hasVersion()) {
        version = metaDataProto.getVersion();
    }
    for (RecordMetaDataProto.JoinedRecordType joinedProto : metaDataProto.getJoinedRecordTypesList()) {
        JoinedRecordTypeBuilder typeBuilder = new JoinedRecordTypeBuilder(joinedProto, this);
        syntheticRecordTypes.put(typeBuilder.getName(), typeBuilder);
    }
}