Examples with RecordMetaData com.apple.foundationdb.record.RecordMetaData used on opensource projects

Example 11 with RecordMetaData

use of com.apple.foundationdb.record.RecordMetaData in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method checkRebuildIndexes.

private CompletableFuture<Void> checkRebuildIndexes(@Nullable UserVersionChecker userVersionChecker, @Nonnull RecordMetaDataProto.DataStoreInfo.Builder info, int oldFormatVersion, @Nonnull RecordMetaData metaData, int oldMetaDataVersion, boolean rebuildRecordCounts, List<CompletableFuture<Void>> work) {
    final boolean newStore = oldFormatVersion == 0;
    final Map<Index, List<RecordType>> indexes = metaData.getIndexesToBuildSince(oldMetaDataVersion);
    if (!indexes.isEmpty()) {
        // If all the new indexes are only for a record type whose primary key has a type prefix, then we can scan less.
        RecordType singleRecordTypeWithPrefixKey = singleRecordTypeWithPrefixKey(indexes);
        final AtomicLong recordCountRef = new AtomicLong(-1);
        final Supplier<CompletableFuture<Long>> lazyRecordCount = getAndRememberFutureLong(recordCountRef, () -> getRecordCountForRebuildIndexes(newStore, rebuildRecordCounts, indexes, singleRecordTypeWithPrefixKey));
        AtomicLong recordsSizeRef = new AtomicLong(-1);
        final Supplier<CompletableFuture<Long>> lazyRecordsSize = getAndRememberFutureLong(recordsSizeRef, () -> getRecordSizeForRebuildIndexes(singleRecordTypeWithPrefixKey));
        if (singleRecordTypeWithPrefixKey == null && formatVersion >= SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION && omitUnsplitRecordSuffix) {
            // Check to see if the unsplit format can be upgraded on an empty store.
            // Only works if singleRecordTypeWithPrefixKey is null as otherwise, the recordCount will not contain
            // all records
            work.add(lazyRecordCount.get().thenAccept(recordCount -> {
                if (recordCount == 0) {
                    if (newStore ? LOGGER.isDebugEnabled() : LOGGER.isInfoEnabled()) {
                        KeyValueLogMessage msg = KeyValueLogMessage.build("upgrading unsplit format on empty store", LogMessageKeys.NEW_FORMAT_VERSION, formatVersion, subspaceProvider.logKey(), subspaceProvider.toString(context));
                        if (newStore) {
                            LOGGER.debug(msg.toString());
                        } else {
                            LOGGER.info(msg.toString());
                        }
                    }
                    omitUnsplitRecordSuffix = formatVersion < SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION;
                    info.clearOmitUnsplitRecordSuffix();
                    // We used snapshot to determine emptiness, and are now acting on it.
                    addRecordsReadConflict();
                }
            }));
        }
        Map<Index, CompletableFuture<IndexState>> newStates = getStatesForRebuildIndexes(userVersionChecker, indexes, lazyRecordCount, lazyRecordsSize, newStore, oldMetaDataVersion, oldFormatVersion);
        return rebuildIndexes(indexes, newStates, work, newStore ? RebuildIndexReason.NEW_STORE : RebuildIndexReason.FEW_RECORDS, oldMetaDataVersion).thenRun(() -> {
            // Log after checking all index states
            maybeLogIndexesNeedingRebuilding(newStates, recordCountRef, recordsSizeRef, rebuildRecordCounts, newStore);
            context.increment(FDBStoreTimer.Counts.INDEXES_NEED_REBUILDING, newStates.entrySet().size());
        });
    } else {
        return work.isEmpty() ? AsyncUtil.DONE : AsyncUtil.whenAll(work);
    }
}

Example 12 with RecordMetaData

use of com.apple.foundationdb.record.RecordMetaData in project fdb-record-layer by FoundationDB.

the class IndexingBase method validateSameMetadataOrThrow.

protected void validateSameMetadataOrThrow(FDBRecordStore store) {
    final RecordMetaData metaData = store.getRecordMetaData();
    final RecordMetaDataProvider recordMetaDataProvider = common.getRecordStoreBuilder().getMetaDataProvider();
    if (recordMetaDataProvider == null || !metaData.equals(recordMetaDataProvider.getRecordMetaData())) {
        throw new MetaDataException("Store does not have the same metadata");
    }
}

Example 13 with RecordMetaData

use of com.apple.foundationdb.record.RecordMetaData in project fdb-record-layer by FoundationDB.

the class FDBMetaDataStore method saveAndSetCurrent.

/**
 * Build meta-data to use from Protobuf and save.
 * @param metaDataProto the Protobuf form of the meta-data to save
 * @return a future that completes when the save is done
 */
@Nonnull
public CompletableFuture<Void> saveAndSetCurrent(@Nonnull RecordMetaDataProto.MetaData metaDataProto) {
    RecordMetaData validatedMetaData = buildMetaData(metaDataProto, true);
    // Load even if not maintaining history so as to get compatibility upgrade before (over-)writing.
    CompletableFuture<Void> future = loadCurrentSerialized().thenApply(oldSerialized -> {
        if (oldSerialized != null) {
            RecordMetaDataProto.MetaData oldProto = parseMetaDataProto(oldSerialized);
            int oldVersion = oldProto.getVersion();
            if (metaDataProto.getVersion() <= oldVersion) {
                LOGGER.warn(KeyValueLogMessage.of("Meta-data version did not increase", subspaceProvider.logKey(), subspaceProvider.toString(context), LogMessageKeys.OLD, oldVersion, LogMessageKeys.NEW, metaDataProto.getVersion()));
                throw new MetaDataException("meta-data version must increase");
            }
            // Build the meta-data, but don't use the local file descriptor as this should validate the original descriptors
            // against each other.
            RecordMetaData oldMetaData = buildMetaData(oldProto, true, false);
            RecordMetaData newMetaData = buildMetaData(metaDataProto, true, false);
            evolutionValidator.validate(oldMetaData, newMetaData);
            if (maintainHistory) {
                SplitHelper.saveWithSplit(context, getSubspace(), HISTORY_KEY_PREFIX.add(oldVersion), oldSerialized, null);
            }
        }
        return null;
    });
    future = future.thenApply(vignore -> {
        recordMetaData = validatedMetaData;
        byte[] serialized = metaDataProto.toByteArray();
        SplitHelper.saveWithSplit(context, getSubspace(), CURRENT_KEY, serialized, null);
        if (cache != null) {
            cache.setCurrentVersion(context, recordMetaData.getVersion());
            addPendingCacheUpdate(recordMetaData);
            addPendingCacheUpdate(serialized);
        }
        return null;
    });
    return instrument(FDBStoreTimer.Events.SAVE_META_DATA, future);
}

Example 14 with RecordMetaData

use of com.apple.foundationdb.record.RecordMetaData in project fdb-record-layer by FoundationDB.

the class JoinedRecordTypeBuilder method build.

@Nonnull
@Override
public JoinedRecordType build(@Nonnull RecordMetaData metaData, @Nonnull Descriptors.FileDescriptor fileDescriptor) {
    final List<JoinedRecordType.JoinConstituent> builtConstituents = getConstituents().stream().map(constituent -> constituent.build(metaData)).collect(Collectors.toList());
    final Descriptors.Descriptor descriptor = fileDescriptor.findMessageTypeByName(name);
    final KeyExpression primaryKey = buildPrimaryKey();
    final Map<String, JoinedRecordType.JoinConstituent> constituentsByName = builtConstituents.stream().collect(Collectors.toMap(JoinedRecordType.Constituent::getName, Function.identity()));
    final List<JoinedRecordType.Join> builtJoins = joins.stream().map(join -> join.build(constituentsByName)).collect(Collectors.toList());
    return new JoinedRecordType(metaData, descriptor, primaryKey, recordTypeKey, indexes, multiTypeIndexes, builtConstituents, builtJoins);
}

Example 15 with RecordMetaData

use of com.apple.foundationdb.record.RecordMetaData in project fdb-record-layer by FoundationDB.

the class OrderingProperty method fromIndexScanOrCoveringIndexScan.

@Nonnull
private static Optional<Ordering> fromIndexScanOrCoveringIndexScan(@Nonnull final PlanContext context, @Nonnull final RecordQueryPlan plan) {
    final RecordQueryIndexPlan recordQueryIndexPlan;
    if (plan instanceof RecordQueryIndexPlan) {
        recordQueryIndexPlan = (RecordQueryIndexPlan) plan;
    } else if (plan instanceof RecordQueryCoveringIndexPlan) {
        final RecordQueryPlanWithIndex planWithIndex = ((RecordQueryCoveringIndexPlan) plan).getIndexPlan();
        if (planWithIndex instanceof RecordQueryIndexPlan) {
            recordQueryIndexPlan = (RecordQueryIndexPlan) planWithIndex;
        } else {
            return Optional.empty();
        }
    } else {
        return Optional.empty();
    }
    final String indexName = recordQueryIndexPlan.getIndexName();
    final RecordMetaData metaData = context.getMetaData();
    final Index index = metaData.getIndex(indexName);
    final Collection<RecordType> recordTypesForIndex = metaData.recordTypesForIndex(index);
    final KeyExpression commonPrimaryKeyForIndex = RecordMetaData.commonPrimaryKey(recordTypesForIndex);
    final KeyExpression keyExpression = ValueIndexExpansionVisitor.fullKey(index, commonPrimaryKeyForIndex);
    final ScanComparisons scanComparisons = recordQueryIndexPlan.getComparisons();
    return fromKeyAndScanComparisons(keyExpression, scanComparisons, plan.isReverse(), index.isUnique());
}