Examples with FormerIndex com.apple.foundationdb.record.metadata.FormerIndex used on opensource projects

Example 1 with FormerIndex

use of com.apple.foundationdb.record.metadata.FormerIndex in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method checkRebuild.

private CompletableFuture<Void> checkRebuild(@Nullable UserVersionChecker userVersionChecker, @Nonnull RecordMetaDataProto.DataStoreInfo.Builder info, @Nonnull RecordMetaData metaData) {
    final List<CompletableFuture<Void>> work = new LinkedList<>();
    final int oldFormatVersion = info.getFormatVersion();
    if (oldFormatVersion != formatVersion) {
        info.setFormatVersion(formatVersion);
        // attempting to read data, i.e., before we update any indexes.
        if ((oldFormatVersion >= MIN_FORMAT_VERSION && oldFormatVersion < SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION && formatVersion >= SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION && !metaData.isSplitLongRecords())) {
            if (LOGGER.isInfoEnabled()) {
                LOGGER.info(KeyValueLogMessage.of("unsplit records stored at old format", LogMessageKeys.OLD_VERSION, oldFormatVersion, LogMessageKeys.NEW_VERSION, formatVersion, subspaceProvider.logKey(), subspaceProvider.toString(context)));
            }
            info.setOmitUnsplitRecordSuffix(true);
            omitUnsplitRecordSuffix = true;
        }
        if (oldFormatVersion >= MIN_FORMAT_VERSION && oldFormatVersion < SAVE_VERSION_WITH_RECORD_FORMAT_VERSION && metaData.isStoreRecordVersions() && !useOldVersionFormat()) {
            if (LOGGER.isInfoEnabled()) {
                LOGGER.info(KeyValueLogMessage.of("migrating record versions to new format"), LogMessageKeys.OLD_VERSION, oldFormatVersion, LogMessageKeys.NEW_VERSION, formatVersion, subspaceProvider.logKey(), subspaceProvider.toString(context));
            }
            addConvertRecordVersions(work);
        }
    }
    final boolean newStore = oldFormatVersion == 0;
    final int oldMetaDataVersion = newStore ? -1 : info.getMetaDataversion();
    final int newMetaDataVersion = metaData.getVersion();
    final boolean metaDataVersionChanged = oldMetaDataVersion != newMetaDataVersion;
    if (metaDataVersionChanged) {
        // Clear the version table if we are no longer storing record versions.
        if (!metaData.isStoreRecordVersions()) {
            final Transaction tr = ensureContextActive();
            tr.clear(getSubspace().subspace(Tuple.from(RECORD_VERSION_KEY)).range());
        }
        info.setMetaDataversion(newMetaDataVersion);
    }
    final boolean rebuildRecordCounts = checkPossiblyRebuildRecordCounts(metaData, info, work, oldFormatVersion);
    // Done if we just needed to update format version (which might trigger record count rebuild).
    if (!metaDataVersionChanged) {
        return work.isEmpty() ? AsyncUtil.DONE : AsyncUtil.whenReady(work.get(0));
    }
    // Remove former indexes.
    for (FormerIndex formerIndex : metaData.getFormerIndexesSince(oldMetaDataVersion)) {
        removeFormerIndex(formerIndex);
    }
    return checkRebuildIndexes(userVersionChecker, info, oldFormatVersion, metaData, oldMetaDataVersion, rebuildRecordCounts, work);
}

Example 2 with FormerIndex

use of com.apple.foundationdb.record.metadata.FormerIndex in project fdb-record-layer by FoundationDB.

the class RecordMetaData method toProto.

/**
 * Serializes the record meta-data to a <code>MetaData</code> proto message. This operates like
 * {@link #toProto()} except that any dependency in the excluded list is not included in the
 * serialized proto message. If the list is set to {@code null}, then all dependencies will be serialized
 * to the proto message including those that are execluded by default.
 *
 * @param excludedDependencies a list of dependencies not to include in the serialized proto
 * @return the serialized <code>MetaData</code> proto message
 * @throws KeyExpression.SerializationException on any serialization failures
 * @throws MetaDataException if this {@code RecordMetaData} was initialized with a
 *      {@linkplain RecordMetaDataBuilder#setLocalFileDescriptor(Descriptors.FileDescriptor) local file descriptor}
 * @see #toProto()
 */
@Nonnull
@SuppressWarnings("deprecation")
public RecordMetaDataProto.MetaData toProto(@Nullable Descriptors.FileDescriptor[] excludedDependencies) throws KeyExpression.SerializationException {
    if (usesLocalRecordsDescriptor) {
        throw new MetaDataException("cannot serialize meta-data with a local records descriptor to proto");
    }
    RecordMetaDataProto.MetaData.Builder builder = RecordMetaDataProto.MetaData.newBuilder();
    // Set the root records.
    builder.setRecords(recordsDescriptor.toProto());
    // Convert the exclusion list to a map
    Map<String, Descriptors.FileDescriptor> excludeMap = null;
    if (excludedDependencies != null) {
        excludeMap = new HashMap<>(excludedDependencies.length);
        for (Descriptors.FileDescriptor dependency : excludedDependencies) {
            excludeMap.put(dependency.getName(), dependency);
        }
    }
    // Add in the rest of dependencies.
    Map<String, Descriptors.FileDescriptor> allDependencies = new TreeMap<>();
    getDependencies(recordsDescriptor, allDependencies, excludeMap);
    for (Descriptors.FileDescriptor dependency : allDependencies.values()) {
        builder.addDependencies(dependency.toProto());
    }
    // Create builders for each index so that we can then add associated record types (etc.).
    Map<String, RecordMetaDataProto.Index.Builder> indexBuilders = new TreeMap<>();
    for (Map.Entry<String, Index> entry : indexes.entrySet()) {
        indexBuilders.put(entry.getKey(), entry.getValue().toProto().toBuilder());
    }
    for (RecordType recordType : getRecordTypes().values()) {
        // Add this record type to each appropriate index.
        for (Index index : recordType.getIndexes()) {
            indexBuilders.get(index.getName()).addRecordType(recordType.getName());
        }
        for (Index index : recordType.getMultiTypeIndexes()) {
            indexBuilders.get(index.getName()).addRecordType(recordType.getName());
        }
        RecordMetaDataProto.RecordType.Builder typeBuilder = builder.addRecordTypesBuilder().setName(recordType.getName()).setPrimaryKey(recordType.getPrimaryKey().toKeyExpression());
        if (recordType.getSinceVersion() != null) {
            typeBuilder.setSinceVersion(recordType.getSinceVersion());
        }
        if (recordType.hasExplicitRecordTypeKey()) {
            typeBuilder.setExplicitKey(LiteralKeyExpression.toProtoValue(recordType.getExplicitRecordTypeKey()));
        }
    }
    indexBuilders.values().forEach(builder::addIndexes);
    // Add in the former indexes.
    for (FormerIndex formerIndex : getFormerIndexes()) {
        builder.addFormerIndexes(formerIndex.toProto());
    }
    // Add in the final options.
    builder.setSplitLongRecords(splitLongRecords);
    builder.setStoreRecordVersions(storeRecordVersions);
    builder.setVersion(version);
    if (usesSubspaceKeyCounter()) {
        builder.setSubspaceKeyCounter(subspaceKeyCounter);
        builder.setUsesSubspaceKeyCounter(true);
    }
    if (recordCountKey != null) {
        builder.setRecordCountKey(recordCountKey.toKeyExpression());
    }
    for (SyntheticRecordType<?> syntheticRecordType : syntheticRecordTypes.values()) {
        if (syntheticRecordType instanceof JoinedRecordType) {
            builder.addJoinedRecordTypes(((JoinedRecordType) syntheticRecordType).toProto());
        }
    }
    return builder.build();
}

Example 3 with FormerIndex

use of com.apple.foundationdb.record.metadata.FormerIndex in project fdb-record-layer by FoundationDB.

the class RecordMetaDataBuilder method removeIndex.

public void removeIndex(@Nonnull String name) {
    Index index = indexes.remove(name);
    if (index == null) {
        throw new MetaDataException("No index named " + name + " defined");
    }
    for (RecordTypeBuilder recordType : recordTypes.values()) {
        recordType.getIndexes().remove(index);
        recordType.getMultiTypeIndexes().remove(index);
    }
    universalIndexes.remove(name);
    formerIndexes.add(new FormerIndex(index.getSubspaceKey(), index.getAddedVersion(), ++version, name));
}

Example 4 with FormerIndex

use of com.apple.foundationdb.record.metadata.FormerIndex in project fdb-record-layer by FoundationDB.

the class RecordMetaDataBuilder method loadProtoExceptRecords.

@SuppressWarnings("deprecation")
private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDataProto) {
    for (RecordMetaDataProto.Index indexProto : metaDataProto.getIndexesList()) {
        List<RecordTypeBuilder> recordTypeBuilders = new ArrayList<>(indexProto.getRecordTypeCount());
        for (String recordTypeName : indexProto.getRecordTypeList()) {
            recordTypeBuilders.add(getRecordType(recordTypeName));
        }
        try {
            addMultiTypeIndex(recordTypeBuilders, new Index(indexProto));
        } catch (KeyExpression.DeserializationException e) {
            throw new MetaDataProtoDeserializationException(e);
        }
    }
    for (RecordMetaDataProto.RecordType typeProto : metaDataProto.getRecordTypesList()) {
        RecordTypeBuilder typeBuilder = getRecordType(typeProto.getName());
        if (typeProto.hasPrimaryKey()) {
            try {
                typeBuilder.setPrimaryKey(KeyExpression.fromProto(typeProto.getPrimaryKey()));
            } catch (KeyExpression.DeserializationException e) {
                throw new MetaDataProtoDeserializationException(e);
            }
        }
        if (typeProto.hasSinceVersion()) {
            typeBuilder.setSinceVersion(typeProto.getSinceVersion());
        }
        if (typeProto.hasExplicitKey()) {
            typeBuilder.setRecordTypeKey(LiteralKeyExpression.fromProtoValue(typeProto.getExplicitKey()));
        }
    }
    if (metaDataProto.hasSplitLongRecords()) {
        splitLongRecords = metaDataProto.getSplitLongRecords();
    }
    if (metaDataProto.hasStoreRecordVersions()) {
        storeRecordVersions = metaDataProto.getStoreRecordVersions();
    }
    for (RecordMetaDataProto.FormerIndex formerIndex : metaDataProto.getFormerIndexesList()) {
        formerIndexes.add(new FormerIndex(formerIndex));
    }
    if (metaDataProto.hasRecordCountKey()) {
        try {
            recordCountKey = KeyExpression.fromProto(metaDataProto.getRecordCountKey());
        } catch (KeyExpression.DeserializationException e) {
            throw new MetaDataProtoDeserializationException(e);
        }
    }
    if (metaDataProto.hasVersion()) {
        version = metaDataProto.getVersion();
    }
    for (RecordMetaDataProto.JoinedRecordType joinedProto : metaDataProto.getJoinedRecordTypesList()) {
        JoinedRecordTypeBuilder typeBuilder = new JoinedRecordTypeBuilder(joinedProto, this);
        syntheticRecordTypes.put(typeBuilder.getName(), typeBuilder);
    }
}