Examples with RecordType com.apple.foundationdb.record.metadata.RecordType used on opensource projects

Example 11 with RecordType

use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method checkRebuildIndexes.

private CompletableFuture<Void> checkRebuildIndexes(@Nullable UserVersionChecker userVersionChecker, @Nonnull RecordMetaDataProto.DataStoreInfo.Builder info, int oldFormatVersion, @Nonnull RecordMetaData metaData, int oldMetaDataVersion, boolean rebuildRecordCounts, List<CompletableFuture<Void>> work) {
    final boolean newStore = oldFormatVersion == 0;
    final Map<Index, List<RecordType>> indexes = metaData.getIndexesToBuildSince(oldMetaDataVersion);
    if (!indexes.isEmpty()) {
        // If all the new indexes are only for a record type whose primary key has a type prefix, then we can scan less.
        RecordType singleRecordTypeWithPrefixKey = singleRecordTypeWithPrefixKey(indexes);
        final AtomicLong recordCountRef = new AtomicLong(-1);
        final Supplier<CompletableFuture<Long>> lazyRecordCount = getAndRememberFutureLong(recordCountRef, () -> getRecordCountForRebuildIndexes(newStore, rebuildRecordCounts, indexes, singleRecordTypeWithPrefixKey));
        AtomicLong recordsSizeRef = new AtomicLong(-1);
        final Supplier<CompletableFuture<Long>> lazyRecordsSize = getAndRememberFutureLong(recordsSizeRef, () -> getRecordSizeForRebuildIndexes(singleRecordTypeWithPrefixKey));
        if (singleRecordTypeWithPrefixKey == null && formatVersion >= SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION && omitUnsplitRecordSuffix) {
            // Check to see if the unsplit format can be upgraded on an empty store.
            // Only works if singleRecordTypeWithPrefixKey is null as otherwise, the recordCount will not contain
            // all records
            work.add(lazyRecordCount.get().thenAccept(recordCount -> {
                if (recordCount == 0) {
                    if (newStore ? LOGGER.isDebugEnabled() : LOGGER.isInfoEnabled()) {
                        KeyValueLogMessage msg = KeyValueLogMessage.build("upgrading unsplit format on empty store", LogMessageKeys.NEW_FORMAT_VERSION, formatVersion, subspaceProvider.logKey(), subspaceProvider.toString(context));
                        if (newStore) {
                            LOGGER.debug(msg.toString());
                        } else {
                            LOGGER.info(msg.toString());
                        }
                    }
                    omitUnsplitRecordSuffix = formatVersion < SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION;
                    info.clearOmitUnsplitRecordSuffix();
                    // We used snapshot to determine emptiness, and are now acting on it.
                    addRecordsReadConflict();
                }
            }));
        }
        Map<Index, CompletableFuture<IndexState>> newStates = getStatesForRebuildIndexes(userVersionChecker, indexes, lazyRecordCount, lazyRecordsSize, newStore, oldMetaDataVersion, oldFormatVersion);
        return rebuildIndexes(indexes, newStates, work, newStore ? RebuildIndexReason.NEW_STORE : RebuildIndexReason.FEW_RECORDS, oldMetaDataVersion).thenRun(() -> {
            // Log after checking all index states
            maybeLogIndexesNeedingRebuilding(newStates, recordCountRef, recordsSizeRef, rebuildRecordCounts, newStore);
            context.increment(FDBStoreTimer.Counts.INDEXES_NEED_REBUILDING, newStates.entrySet().size());
        });
    } else {
        return work.isEmpty() ? AsyncUtil.DONE : AsyncUtil.whenAll(work);
    }
}

Example 12 with RecordType

use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.

the class RecordMetaData method commonPrimaryKey.

@Nullable
public static KeyExpression commonPrimaryKey(@Nonnull Collection<RecordType> recordTypes) {
    KeyExpression common = null;
    boolean first = true;
    for (RecordType recordType : recordTypes) {
        if (first) {
            common = recordType.getPrimaryKey();
            first = false;
        } else if (!common.equals(recordType.getPrimaryKey())) {
            return null;
        }
    }
    return common;
}

Example 13 with RecordType

use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.

the class RecordMetaData method toProto.

/**
 * Serializes the record meta-data to a <code>MetaData</code> proto message. This operates like
 * {@link #toProto()} except that any dependency in the excluded list is not included in the
 * serialized proto message. If the list is set to {@code null}, then all dependencies will be serialized
 * to the proto message including those that are execluded by default.
 *
 * @param excludedDependencies a list of dependencies not to include in the serialized proto
 * @return the serialized <code>MetaData</code> proto message
 * @throws KeyExpression.SerializationException on any serialization failures
 * @throws MetaDataException if this {@code RecordMetaData} was initialized with a
 *      {@linkplain RecordMetaDataBuilder#setLocalFileDescriptor(Descriptors.FileDescriptor) local file descriptor}
 * @see #toProto()
 */
@Nonnull
@SuppressWarnings("deprecation")
public RecordMetaDataProto.MetaData toProto(@Nullable Descriptors.FileDescriptor[] excludedDependencies) throws KeyExpression.SerializationException {
    if (usesLocalRecordsDescriptor) {
        throw new MetaDataException("cannot serialize meta-data with a local records descriptor to proto");
    }
    RecordMetaDataProto.MetaData.Builder builder = RecordMetaDataProto.MetaData.newBuilder();
    // Set the root records.
    builder.setRecords(recordsDescriptor.toProto());
    // Convert the exclusion list to a map
    Map<String, Descriptors.FileDescriptor> excludeMap = null;
    if (excludedDependencies != null) {
        excludeMap = new HashMap<>(excludedDependencies.length);
        for (Descriptors.FileDescriptor dependency : excludedDependencies) {
            excludeMap.put(dependency.getName(), dependency);
        }
    }
    // Add in the rest of dependencies.
    Map<String, Descriptors.FileDescriptor> allDependencies = new TreeMap<>();
    getDependencies(recordsDescriptor, allDependencies, excludeMap);
    for (Descriptors.FileDescriptor dependency : allDependencies.values()) {
        builder.addDependencies(dependency.toProto());
    }
    // Create builders for each index so that we can then add associated record types (etc.).
    Map<String, RecordMetaDataProto.Index.Builder> indexBuilders = new TreeMap<>();
    for (Map.Entry<String, Index> entry : indexes.entrySet()) {
        indexBuilders.put(entry.getKey(), entry.getValue().toProto().toBuilder());
    }
    for (RecordType recordType : getRecordTypes().values()) {
        // Add this record type to each appropriate index.
        for (Index index : recordType.getIndexes()) {
            indexBuilders.get(index.getName()).addRecordType(recordType.getName());
        }
        for (Index index : recordType.getMultiTypeIndexes()) {
            indexBuilders.get(index.getName()).addRecordType(recordType.getName());
        }
        RecordMetaDataProto.RecordType.Builder typeBuilder = builder.addRecordTypesBuilder().setName(recordType.getName()).setPrimaryKey(recordType.getPrimaryKey().toKeyExpression());
        if (recordType.getSinceVersion() != null) {
            typeBuilder.setSinceVersion(recordType.getSinceVersion());
        }
        if (recordType.hasExplicitRecordTypeKey()) {
            typeBuilder.setExplicitKey(LiteralKeyExpression.toProtoValue(recordType.getExplicitRecordTypeKey()));
        }
    }
    indexBuilders.values().forEach(builder::addIndexes);
    // Add in the former indexes.
    for (FormerIndex formerIndex : getFormerIndexes()) {
        builder.addFormerIndexes(formerIndex.toProto());
    }
    // Add in the final options.
    builder.setSplitLongRecords(splitLongRecords);
    builder.setStoreRecordVersions(storeRecordVersions);
    builder.setVersion(version);
    if (usesSubspaceKeyCounter()) {
        builder.setSubspaceKeyCounter(subspaceKeyCounter);
        builder.setUsesSubspaceKeyCounter(true);
    }
    if (recordCountKey != null) {
        builder.setRecordCountKey(recordCountKey.toKeyExpression());
    }
    for (SyntheticRecordType<?> syntheticRecordType : syntheticRecordTypes.values()) {
        if (syntheticRecordType instanceof JoinedRecordType) {
            builder.addJoinedRecordTypes(((JoinedRecordType) syntheticRecordType).toProto());
        }
    }
    return builder.build();
}

Example 14 with RecordType

use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.

the class OrderingProperty method fromIndexScanOrCoveringIndexScan.

@Nonnull
private static Optional<Ordering> fromIndexScanOrCoveringIndexScan(@Nonnull final PlanContext context, @Nonnull final RecordQueryPlan plan) {
    final RecordQueryIndexPlan recordQueryIndexPlan;
    if (plan instanceof RecordQueryIndexPlan) {
        recordQueryIndexPlan = (RecordQueryIndexPlan) plan;
    } else if (plan instanceof RecordQueryCoveringIndexPlan) {
        final RecordQueryPlanWithIndex planWithIndex = ((RecordQueryCoveringIndexPlan) plan).getIndexPlan();
        if (planWithIndex instanceof RecordQueryIndexPlan) {
            recordQueryIndexPlan = (RecordQueryIndexPlan) planWithIndex;
        } else {
            return Optional.empty();
        }
    } else {
        return Optional.empty();
    }
    final String indexName = recordQueryIndexPlan.getIndexName();
    final RecordMetaData metaData = context.getMetaData();
    final Index index = metaData.getIndex(indexName);
    final Collection<RecordType> recordTypesForIndex = metaData.recordTypesForIndex(index);
    final KeyExpression commonPrimaryKeyForIndex = RecordMetaData.commonPrimaryKey(recordTypesForIndex);
    final KeyExpression keyExpression = ValueIndexExpansionVisitor.fullKey(index, commonPrimaryKeyForIndex);
    final ScanComparisons scanComparisons = recordQueryIndexPlan.getComparisons();
    return fromKeyAndScanComparisons(keyExpression, scanComparisons, plan.isReverse(), index.isUnique());
}

Example 15 with RecordType

use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.

the class RecordQueryPlannerSubstitutionVisitor method availableFields.

@Nonnull
public static AvailableFields availableFields(@Nonnull RecordMetaData recordMetaData, @Nonnull PlannableIndexTypes indexTypes, @Nullable KeyExpression commonPrimaryKey, @Nonnull RecordQueryPlan plan) {
    if (plan instanceof RecordQueryPlanWithIndex) {
        RecordQueryPlanWithIndex indexPlan = (RecordQueryPlanWithIndex) plan;
        Index index = recordMetaData.getIndex(indexPlan.getIndexName());
        final Collection<RecordType> recordTypes = recordMetaData.recordTypesForIndex(index);
        if (recordTypes.size() != 1) {
            return AvailableFields.NO_FIELDS;
        }
        final RecordType recordType = Iterables.getOnlyElement(recordTypes);
        return AvailableFields.fromIndex(recordType, index, indexTypes, commonPrimaryKey);
    } else if (plan instanceof RecordQueryFetchFromPartialRecordPlan) {
        RecordQueryFetchFromPartialRecordPlan fetchPlan = (RecordQueryFetchFromPartialRecordPlan) plan;
        return fetchPlan.getChild().getAvailableFields();
    }
    return AvailableFields.NO_FIELDS;
}