Search in sources :

Example 1 with SyntheticRecordType

use of com.apple.foundationdb.record.metadata.SyntheticRecordType in project fdb-record-layer by FoundationDB.

the class SyntheticRecordPlanner method forIndex.

/**
 * Construct a plan for generating synthetic records for a given index.
 *
 * The generated records will be of indexed record types.
 *
 * Used by the {@link com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer} to build from a full scan of stored records.
 * @param index an index on synthetic record types
 * @return a plan that can be applied to scanned records to generate synthetic records
 */
@Nonnull
public SyntheticRecordFromStoredRecordPlan forIndex(@Nonnull Index index) {
    final Collection<RecordType> recordTypes = recordMetaData.recordTypesForIndex(index);
    if (recordTypes.size() == 1) {
        final RecordType recordType = recordTypes.iterator().next();
        if (!recordType.isSynthetic()) {
            throw new RecordCoreException("Index does not apply to synthetic record types " + index);
        }
        return forType((SyntheticRecordType<?>) recordType);
    }
    Multimap<String, SyntheticRecordFromStoredRecordPlan> byType = ArrayListMultimap.create();
    for (RecordType recordType : recordTypes) {
        if (!(recordType instanceof JoinedRecordType)) {
            throw unknownSyntheticType(recordType);
        }
        JoinedRecordType joinedRecordType = (JoinedRecordType) recordType;
        Optional<JoinedRecordType.JoinConstituent> maybeConstituent = joinedRecordType.getConstituents().stream().filter(c -> !c.isOuterJoined()).findFirst();
        if (maybeConstituent.isPresent()) {
            addToByType(byType, joinedRecordType, maybeConstituent.get());
        } else {
            for (JoinedRecordType.JoinConstituent joinConstituent : joinedRecordType.getConstituents()) {
                addToByType(byType, joinedRecordType, joinConstituent);
            }
        }
    }
    return createByType(byType);
}
Also used : ArrayListMultimap(com.google.common.collect.ArrayListMultimap) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) HashMap(java.util.HashMap) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) Multimap(com.google.common.collect.Multimap) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) ArrayList(java.util.ArrayList) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) HashSet(java.util.HashSet) JoinedRecordType(com.apple.foundationdb.record.metadata.JoinedRecordType) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) Map(java.util.Map) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) Collection(java.util.Collection) Set(java.util.Set) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) List(java.util.List) RecordType(com.apple.foundationdb.record.metadata.RecordType) SyntheticRecordType(com.apple.foundationdb.record.metadata.SyntheticRecordType) Index(com.apple.foundationdb.record.metadata.Index) RecordStoreState(com.apple.foundationdb.record.RecordStoreState) Optional(java.util.Optional) API(com.apple.foundationdb.annotation.API) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) JoinedRecordType(com.apple.foundationdb.record.metadata.JoinedRecordType) RecordType(com.apple.foundationdb.record.metadata.RecordType) SyntheticRecordType(com.apple.foundationdb.record.metadata.SyntheticRecordType) JoinedRecordType(com.apple.foundationdb.record.metadata.JoinedRecordType) Nonnull(javax.annotation.Nonnull)

Example 2 with SyntheticRecordType

use of com.apple.foundationdb.record.metadata.SyntheticRecordType in project fdb-record-layer by FoundationDB.

the class RecordMetaData method toProto.

/**
 * Serializes the record meta-data to a <code>MetaData</code> proto message. This operates like
 * {@link #toProto()} except that any dependency in the excluded list is not included in the
 * serialized proto message. If the list is set to {@code null}, then all dependencies will be serialized
 * to the proto message including those that are execluded by default.
 *
 * @param excludedDependencies a list of dependencies not to include in the serialized proto
 * @return the serialized <code>MetaData</code> proto message
 * @throws KeyExpression.SerializationException on any serialization failures
 * @throws MetaDataException if this {@code RecordMetaData} was initialized with a
 *      {@linkplain RecordMetaDataBuilder#setLocalFileDescriptor(Descriptors.FileDescriptor) local file descriptor}
 * @see #toProto()
 */
@Nonnull
@SuppressWarnings("deprecation")
public RecordMetaDataProto.MetaData toProto(@Nullable Descriptors.FileDescriptor[] excludedDependencies) throws KeyExpression.SerializationException {
    if (usesLocalRecordsDescriptor) {
        throw new MetaDataException("cannot serialize meta-data with a local records descriptor to proto");
    }
    RecordMetaDataProto.MetaData.Builder builder = RecordMetaDataProto.MetaData.newBuilder();
    // Set the root records.
    builder.setRecords(recordsDescriptor.toProto());
    // Convert the exclusion list to a map
    Map<String, Descriptors.FileDescriptor> excludeMap = null;
    if (excludedDependencies != null) {
        excludeMap = new HashMap<>(excludedDependencies.length);
        for (Descriptors.FileDescriptor dependency : excludedDependencies) {
            excludeMap.put(dependency.getName(), dependency);
        }
    }
    // Add in the rest of dependencies.
    Map<String, Descriptors.FileDescriptor> allDependencies = new TreeMap<>();
    getDependencies(recordsDescriptor, allDependencies, excludeMap);
    for (Descriptors.FileDescriptor dependency : allDependencies.values()) {
        builder.addDependencies(dependency.toProto());
    }
    // Create builders for each index so that we can then add associated record types (etc.).
    Map<String, RecordMetaDataProto.Index.Builder> indexBuilders = new TreeMap<>();
    for (Map.Entry<String, Index> entry : indexes.entrySet()) {
        indexBuilders.put(entry.getKey(), entry.getValue().toProto().toBuilder());
    }
    for (RecordType recordType : getRecordTypes().values()) {
        // Add this record type to each appropriate index.
        for (Index index : recordType.getIndexes()) {
            indexBuilders.get(index.getName()).addRecordType(recordType.getName());
        }
        for (Index index : recordType.getMultiTypeIndexes()) {
            indexBuilders.get(index.getName()).addRecordType(recordType.getName());
        }
        RecordMetaDataProto.RecordType.Builder typeBuilder = builder.addRecordTypesBuilder().setName(recordType.getName()).setPrimaryKey(recordType.getPrimaryKey().toKeyExpression());
        if (recordType.getSinceVersion() != null) {
            typeBuilder.setSinceVersion(recordType.getSinceVersion());
        }
        if (recordType.hasExplicitRecordTypeKey()) {
            typeBuilder.setExplicitKey(LiteralKeyExpression.toProtoValue(recordType.getExplicitRecordTypeKey()));
        }
    }
    indexBuilders.values().forEach(builder::addIndexes);
    // Add in the former indexes.
    for (FormerIndex formerIndex : getFormerIndexes()) {
        builder.addFormerIndexes(formerIndex.toProto());
    }
    // Add in the final options.
    builder.setSplitLongRecords(splitLongRecords);
    builder.setStoreRecordVersions(storeRecordVersions);
    builder.setVersion(version);
    if (usesSubspaceKeyCounter()) {
        builder.setSubspaceKeyCounter(subspaceKeyCounter);
        builder.setUsesSubspaceKeyCounter(true);
    }
    if (recordCountKey != null) {
        builder.setRecordCountKey(recordCountKey.toKeyExpression());
    }
    for (SyntheticRecordType<?> syntheticRecordType : syntheticRecordTypes.values()) {
        if (syntheticRecordType instanceof JoinedRecordType) {
            builder.addJoinedRecordTypes(((JoinedRecordType) syntheticRecordType).toProto());
        }
    }
    return builder.build();
}
Also used : JoinedRecordType(com.apple.foundationdb.record.metadata.JoinedRecordType) FormerIndex(com.apple.foundationdb.record.metadata.FormerIndex) Index(com.apple.foundationdb.record.metadata.Index) TreeMap(java.util.TreeMap) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) FormerIndex(com.apple.foundationdb.record.metadata.FormerIndex) JoinedRecordType(com.apple.foundationdb.record.metadata.JoinedRecordType) RecordType(com.apple.foundationdb.record.metadata.RecordType) SyntheticRecordType(com.apple.foundationdb.record.metadata.SyntheticRecordType) Descriptors(com.google.protobuf.Descriptors) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) TreeMap(java.util.TreeMap) Map(java.util.Map) Nonnull(javax.annotation.Nonnull)

Example 3 with SyntheticRecordType

use of com.apple.foundationdb.record.metadata.SyntheticRecordType in project fdb-record-layer by FoundationDB.

the class RecordMetaDataBuilder method build.

/**
 * Build and validate meta-data with specific index registry.
 * @param validate {@code true} to validate the new meta-data
 * @return new meta-data
 */
@Nonnull
public RecordMetaData build(boolean validate) {
    Map<String, RecordType> builtRecordTypes = Maps.newHashMapWithExpectedSize(recordTypes.size());
    Map<String, SyntheticRecordType<?>> builtSyntheticRecordTypes = Maps.newHashMapWithExpectedSize(syntheticRecordTypes.size());
    RecordMetaData metaData = new RecordMetaData(recordsDescriptor, getUnionDescriptor(), unionFields, builtRecordTypes, builtSyntheticRecordTypes, indexes, universalIndexes, formerIndexes, splitLongRecords, storeRecordVersions, version, subspaceKeyCounter, usesSubspaceKeyCounter, recordCountKey, localFileDescriptor != null);
    for (RecordTypeBuilder recordTypeBuilder : recordTypes.values()) {
        KeyExpression primaryKey = recordTypeBuilder.getPrimaryKey();
        if (primaryKey != null) {
            builtRecordTypes.put(recordTypeBuilder.getName(), recordTypeBuilder.build(metaData));
            for (Index index : recordTypeBuilder.getIndexes()) {
                index.setPrimaryKeyComponentPositions(buildPrimaryKeyComponentPositions(index.getRootExpression(), primaryKey));
            }
        } else {
            throw new MetaDataException("Record type " + recordTypeBuilder.getName() + " must have a primary key");
        }
    }
    if (!syntheticRecordTypes.isEmpty()) {
        DescriptorProtos.FileDescriptorProto.Builder fileBuilder = DescriptorProtos.FileDescriptorProto.newBuilder();
        fileBuilder.setName("_synthetic");
        fileBuilder.addDependency(unionDescriptor.getFile().getName());
        syntheticRecordTypes.values().forEach(recordTypeBuilder -> recordTypeBuilder.buildDescriptor(fileBuilder));
        final Descriptors.FileDescriptor fileDescriptor;
        try {
            final Descriptors.FileDescriptor[] dependencies = new Descriptors.FileDescriptor[] { unionDescriptor.getFile() };
            fileDescriptor = Descriptors.FileDescriptor.buildFrom(fileBuilder.build(), dependencies);
        } catch (Descriptors.DescriptorValidationException ex) {
            throw new MetaDataException("Could not build synthesized file descriptor", ex);
        }
        for (SyntheticRecordTypeBuilder<?> recordTypeBuilder : syntheticRecordTypes.values()) {
            builtSyntheticRecordTypes.put(recordTypeBuilder.getName(), recordTypeBuilder.build(metaData, fileDescriptor));
        }
    }
    if (validate) {
        final MetaDataValidator validator = new MetaDataValidator(metaData, indexMaintainerRegistry);
        validator.validate();
    }
    return metaData;
}
Also used : SyntheticRecordType(com.apple.foundationdb.record.metadata.SyntheticRecordType) LiteralKeyExpression(com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression) FieldKeyExpression(com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) FormerIndex(com.apple.foundationdb.record.metadata.FormerIndex) Index(com.apple.foundationdb.record.metadata.Index) SyntheticRecordTypeBuilder(com.apple.foundationdb.record.metadata.SyntheticRecordTypeBuilder) RecordTypeBuilder(com.apple.foundationdb.record.metadata.RecordTypeBuilder) JoinedRecordTypeBuilder(com.apple.foundationdb.record.metadata.JoinedRecordTypeBuilder) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) RecordType(com.apple.foundationdb.record.metadata.RecordType) SyntheticRecordType(com.apple.foundationdb.record.metadata.SyntheticRecordType) Descriptors(com.google.protobuf.Descriptors) MetaDataValidator(com.apple.foundationdb.record.metadata.MetaDataValidator) Nonnull(javax.annotation.Nonnull)

Example 4 with SyntheticRecordType

use of com.apple.foundationdb.record.metadata.SyntheticRecordType in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method loadSyntheticRecord.

/**
 * Load a {@link FDBSyntheticRecord synthetic record} by loading its stored constituent records and synthesizing it from them.
 * @param primaryKey the primary key of the synthetic record, which includes the primary keys of the constituents
 * @return a future which completes to the synthesized record
 */
@Nonnull
@API(API.Status.EXPERIMENTAL)
public CompletableFuture<FDBSyntheticRecord> loadSyntheticRecord(@Nonnull Tuple primaryKey) {
    SyntheticRecordType<?> syntheticRecordType = getRecordMetaData().getSyntheticRecordTypeFromRecordTypeKey(primaryKey.get(0));
    int nconstituents = syntheticRecordType.getConstituents().size();
    if (nconstituents != primaryKey.size() - 1) {
        throw recordCoreException("Primary key does not have correct number of nested keys: " + primaryKey);
    }
    final Map<String, FDBStoredRecord<? extends Message>> constituents = new ConcurrentHashMap<>(nconstituents);
    final CompletableFuture<?>[] futures = new CompletableFuture<?>[nconstituents];
    for (int i = 0; i < nconstituents; i++) {
        final SyntheticRecordType.Constituent constituent = syntheticRecordType.getConstituents().get(i);
        final Tuple constituentKey = primaryKey.getNestedTuple(i + 1);
        if (constituentKey == null) {
            futures[i] = AsyncUtil.DONE;
        } else {
            futures[i] = loadRecordAsync(constituentKey).thenApply(rec -> {
                if (rec == null) {
                    throw new RecordDoesNotExistException("constituent record not found: " + constituent.getName());
                }
                constituents.put(constituent.getName(), rec);
                return null;
            });
        }
    }
    return CompletableFuture.allOf(futures).thenApply(vignore -> FDBSyntheticRecord.of(syntheticRecordType, constituents));
}
Also used : LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) UnaryOperator(java.util.function.UnaryOperator) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) RecordSerializer(com.apple.foundationdb.record.provider.common.RecordSerializer) Subspace(com.apple.foundationdb.subspace.Subspace) MutationType(com.apple.foundationdb.MutationType) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) Map(java.util.Map) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) QueryToKeyMatcher(com.apple.foundationdb.record.query.QueryToKeyMatcher) InvalidProtocolBufferException(com.google.protobuf.InvalidProtocolBufferException) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) Set(java.util.Set) TupleRange(com.apple.foundationdb.record.TupleRange) KeySpacePath(com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath) ByteOrder(java.nio.ByteOrder) SyntheticRecordType(com.apple.foundationdb.record.metadata.SyntheticRecordType) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) RecordStoreState(com.apple.foundationdb.record.RecordStoreState) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) API(com.apple.foundationdb.annotation.API) FunctionNames(com.apple.foundationdb.record.FunctionNames) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RangeSet(com.apple.foundationdb.async.RangeSet) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Supplier(java.util.function.Supplier) FormerIndex(com.apple.foundationdb.record.metadata.FormerIndex) ArrayList(java.util.ArrayList) ByteScanLimiter(com.apple.foundationdb.record.ByteScanLimiter) ParameterRelationshipGraph(com.apple.foundationdb.record.query.ParameterRelationshipGraph) LoggableException(com.apple.foundationdb.util.LoggableException) CloseableAsyncIterator(com.apple.foundationdb.async.CloseableAsyncIterator) IndexRecordFunction(com.apple.foundationdb.record.metadata.IndexRecordFunction) Nullable(javax.annotation.Nullable) ByteArrayUtil2(com.apple.foundationdb.tuple.ByteArrayUtil2) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) CursorLimitManager(com.apple.foundationdb.record.cursors.CursorLimitManager) ExecuteState(com.apple.foundationdb.record.ExecuteState) AtomicLong(java.util.concurrent.atomic.AtomicLong) RecordType(com.apple.foundationdb.record.metadata.RecordType) Index(com.apple.foundationdb.record.metadata.Index) DynamicMessageRecordSerializer(com.apple.foundationdb.record.provider.common.DynamicMessageRecordSerializer) SyntheticRecordPlanner(com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordPlanner) IndexEntry(com.apple.foundationdb.record.IndexEntry) LoggerFactory(org.slf4j.LoggerFactory) RecordCoreStorageException(com.apple.foundationdb.record.RecordCoreStorageException) ByteBuffer(java.nio.ByteBuffer) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) Transaction(com.apple.foundationdb.Transaction) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) PipelineOperation(com.apple.foundationdb.record.PipelineOperation) RecordMetaDataProto(com.apple.foundationdb.record.RecordMetaDataProto) ByteArrayUtil(com.apple.foundationdb.tuple.ByteArrayUtil) KeyValue(com.apple.foundationdb.KeyValue) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) IndexQueryabilityFilter(com.apple.foundationdb.record.query.IndexQueryabilityFilter) AndComponent(com.apple.foundationdb.record.query.expressions.AndComponent) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) Collectors(java.util.stream.Collectors) ByteString(com.google.protobuf.ByteString) List(java.util.List) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) AggregateFunctionNotSupportedException(com.apple.foundationdb.record.AggregateFunctionNotSupportedException) RecordTypeKeyComparison(com.apple.foundationdb.record.query.expressions.RecordTypeKeyComparison) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Optional(java.util.Optional) MutableRecordStoreState(com.apple.foundationdb.record.MutableRecordStoreState) RecordTypeOrBuilder(com.apple.foundationdb.record.metadata.RecordTypeOrBuilder) SyntheticRecordFromStoredRecordPlan(com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordFromStoredRecordPlan) SpotBugsSuppressWarnings(com.apple.foundationdb.annotation.SpotBugsSuppressWarnings) Descriptors(com.google.protobuf.Descriptors) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) CursorStreamingMode(com.apple.foundationdb.record.CursorStreamingMode) Key(com.apple.foundationdb.record.metadata.Key) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) EndpointType(com.apple.foundationdb.record.EndpointType) ScanProperties(com.apple.foundationdb.record.ScanProperties) Suppliers(com.google.common.base.Suppliers) LinkedList(java.util.LinkedList) Nonnull(javax.annotation.Nonnull) EmptyKeyExpression(com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression) MoreAsyncUtil(com.apple.foundationdb.async.MoreAsyncUtil) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) IndexState(com.apple.foundationdb.record.IndexState) StoreRecordFunction(com.apple.foundationdb.record.metadata.StoreRecordFunction) ReadTransaction(com.apple.foundationdb.ReadTransaction) AsyncIterable(com.apple.foundationdb.async.AsyncIterable) FDBRecordStoreStateCache(com.apple.foundationdb.record.provider.foundationdb.storestate.FDBRecordStoreStateCache) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Message(com.google.protobuf.Message) SyntheticRecordType(com.apple.foundationdb.record.metadata.SyntheticRecordType) ByteString(com.google.protobuf.ByteString) CompletableFuture(java.util.concurrent.CompletableFuture) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull) API(com.apple.foundationdb.annotation.API)

Aggregations

Index (com.apple.foundationdb.record.metadata.Index)4 RecordType (com.apple.foundationdb.record.metadata.RecordType)4 SyntheticRecordType (com.apple.foundationdb.record.metadata.SyntheticRecordType)4 Nonnull (javax.annotation.Nonnull)4 FormerIndex (com.apple.foundationdb.record.metadata.FormerIndex)3 MetaDataException (com.apple.foundationdb.record.metadata.MetaDataException)3 Descriptors (com.google.protobuf.Descriptors)3 API (com.apple.foundationdb.annotation.API)2 RecordCoreArgumentException (com.apple.foundationdb.record.RecordCoreArgumentException)2 RecordCoreException (com.apple.foundationdb.record.RecordCoreException)2 RecordMetaData (com.apple.foundationdb.record.RecordMetaData)2 JoinedRecordType (com.apple.foundationdb.record.metadata.JoinedRecordType)2 KeyExpression (com.apple.foundationdb.record.metadata.expressions.KeyExpression)2 HashMap (java.util.HashMap)2 Map (java.util.Map)2 KeyValue (com.apple.foundationdb.KeyValue)1 MutationType (com.apple.foundationdb.MutationType)1 Range (com.apple.foundationdb.Range)1 ReadTransaction (com.apple.foundationdb.ReadTransaction)1 Transaction (com.apple.foundationdb.Transaction)1