use of com.apple.foundationdb.record.metadata.SyntheticRecordType in project fdb-record-layer by FoundationDB.
the class SyntheticRecordPlanner method forIndex.
/**
* Construct a plan for generating synthetic records for a given index.
*
* The generated records will be of indexed record types.
*
* Used by the {@link com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer} to build from a full scan of stored records.
* @param index an index on synthetic record types
* @return a plan that can be applied to scanned records to generate synthetic records
*/
@Nonnull
public SyntheticRecordFromStoredRecordPlan forIndex(@Nonnull Index index) {
final Collection<RecordType> recordTypes = recordMetaData.recordTypesForIndex(index);
if (recordTypes.size() == 1) {
final RecordType recordType = recordTypes.iterator().next();
if (!recordType.isSynthetic()) {
throw new RecordCoreException("Index does not apply to synthetic record types " + index);
}
return forType((SyntheticRecordType<?>) recordType);
}
Multimap<String, SyntheticRecordFromStoredRecordPlan> byType = ArrayListMultimap.create();
for (RecordType recordType : recordTypes) {
if (!(recordType instanceof JoinedRecordType)) {
throw unknownSyntheticType(recordType);
}
JoinedRecordType joinedRecordType = (JoinedRecordType) recordType;
Optional<JoinedRecordType.JoinConstituent> maybeConstituent = joinedRecordType.getConstituents().stream().filter(c -> !c.isOuterJoined()).findFirst();
if (maybeConstituent.isPresent()) {
addToByType(byType, joinedRecordType, maybeConstituent.get());
} else {
for (JoinedRecordType.JoinConstituent joinConstituent : joinedRecordType.getConstituents()) {
addToByType(byType, joinedRecordType, joinConstituent);
}
}
}
return createByType(byType);
}
use of com.apple.foundationdb.record.metadata.SyntheticRecordType in project fdb-record-layer by FoundationDB.
the class RecordMetaData method toProto.
/**
* Serializes the record meta-data to a <code>MetaData</code> proto message. This operates like
* {@link #toProto()} except that any dependency in the excluded list is not included in the
* serialized proto message. If the list is set to {@code null}, then all dependencies will be serialized
* to the proto message including those that are execluded by default.
*
* @param excludedDependencies a list of dependencies not to include in the serialized proto
* @return the serialized <code>MetaData</code> proto message
* @throws KeyExpression.SerializationException on any serialization failures
* @throws MetaDataException if this {@code RecordMetaData} was initialized with a
* {@linkplain RecordMetaDataBuilder#setLocalFileDescriptor(Descriptors.FileDescriptor) local file descriptor}
* @see #toProto()
*/
@Nonnull
@SuppressWarnings("deprecation")
public RecordMetaDataProto.MetaData toProto(@Nullable Descriptors.FileDescriptor[] excludedDependencies) throws KeyExpression.SerializationException {
if (usesLocalRecordsDescriptor) {
throw new MetaDataException("cannot serialize meta-data with a local records descriptor to proto");
}
RecordMetaDataProto.MetaData.Builder builder = RecordMetaDataProto.MetaData.newBuilder();
// Set the root records.
builder.setRecords(recordsDescriptor.toProto());
// Convert the exclusion list to a map
Map<String, Descriptors.FileDescriptor> excludeMap = null;
if (excludedDependencies != null) {
excludeMap = new HashMap<>(excludedDependencies.length);
for (Descriptors.FileDescriptor dependency : excludedDependencies) {
excludeMap.put(dependency.getName(), dependency);
}
}
// Add in the rest of dependencies.
Map<String, Descriptors.FileDescriptor> allDependencies = new TreeMap<>();
getDependencies(recordsDescriptor, allDependencies, excludeMap);
for (Descriptors.FileDescriptor dependency : allDependencies.values()) {
builder.addDependencies(dependency.toProto());
}
// Create builders for each index so that we can then add associated record types (etc.).
Map<String, RecordMetaDataProto.Index.Builder> indexBuilders = new TreeMap<>();
for (Map.Entry<String, Index> entry : indexes.entrySet()) {
indexBuilders.put(entry.getKey(), entry.getValue().toProto().toBuilder());
}
for (RecordType recordType : getRecordTypes().values()) {
// Add this record type to each appropriate index.
for (Index index : recordType.getIndexes()) {
indexBuilders.get(index.getName()).addRecordType(recordType.getName());
}
for (Index index : recordType.getMultiTypeIndexes()) {
indexBuilders.get(index.getName()).addRecordType(recordType.getName());
}
RecordMetaDataProto.RecordType.Builder typeBuilder = builder.addRecordTypesBuilder().setName(recordType.getName()).setPrimaryKey(recordType.getPrimaryKey().toKeyExpression());
if (recordType.getSinceVersion() != null) {
typeBuilder.setSinceVersion(recordType.getSinceVersion());
}
if (recordType.hasExplicitRecordTypeKey()) {
typeBuilder.setExplicitKey(LiteralKeyExpression.toProtoValue(recordType.getExplicitRecordTypeKey()));
}
}
indexBuilders.values().forEach(builder::addIndexes);
// Add in the former indexes.
for (FormerIndex formerIndex : getFormerIndexes()) {
builder.addFormerIndexes(formerIndex.toProto());
}
// Add in the final options.
builder.setSplitLongRecords(splitLongRecords);
builder.setStoreRecordVersions(storeRecordVersions);
builder.setVersion(version);
if (usesSubspaceKeyCounter()) {
builder.setSubspaceKeyCounter(subspaceKeyCounter);
builder.setUsesSubspaceKeyCounter(true);
}
if (recordCountKey != null) {
builder.setRecordCountKey(recordCountKey.toKeyExpression());
}
for (SyntheticRecordType<?> syntheticRecordType : syntheticRecordTypes.values()) {
if (syntheticRecordType instanceof JoinedRecordType) {
builder.addJoinedRecordTypes(((JoinedRecordType) syntheticRecordType).toProto());
}
}
return builder.build();
}
use of com.apple.foundationdb.record.metadata.SyntheticRecordType in project fdb-record-layer by FoundationDB.
the class RecordMetaDataBuilder method build.
/**
* Build and validate meta-data with specific index registry.
* @param validate {@code true} to validate the new meta-data
* @return new meta-data
*/
@Nonnull
public RecordMetaData build(boolean validate) {
Map<String, RecordType> builtRecordTypes = Maps.newHashMapWithExpectedSize(recordTypes.size());
Map<String, SyntheticRecordType<?>> builtSyntheticRecordTypes = Maps.newHashMapWithExpectedSize(syntheticRecordTypes.size());
RecordMetaData metaData = new RecordMetaData(recordsDescriptor, getUnionDescriptor(), unionFields, builtRecordTypes, builtSyntheticRecordTypes, indexes, universalIndexes, formerIndexes, splitLongRecords, storeRecordVersions, version, subspaceKeyCounter, usesSubspaceKeyCounter, recordCountKey, localFileDescriptor != null);
for (RecordTypeBuilder recordTypeBuilder : recordTypes.values()) {
KeyExpression primaryKey = recordTypeBuilder.getPrimaryKey();
if (primaryKey != null) {
builtRecordTypes.put(recordTypeBuilder.getName(), recordTypeBuilder.build(metaData));
for (Index index : recordTypeBuilder.getIndexes()) {
index.setPrimaryKeyComponentPositions(buildPrimaryKeyComponentPositions(index.getRootExpression(), primaryKey));
}
} else {
throw new MetaDataException("Record type " + recordTypeBuilder.getName() + " must have a primary key");
}
}
if (!syntheticRecordTypes.isEmpty()) {
DescriptorProtos.FileDescriptorProto.Builder fileBuilder = DescriptorProtos.FileDescriptorProto.newBuilder();
fileBuilder.setName("_synthetic");
fileBuilder.addDependency(unionDescriptor.getFile().getName());
syntheticRecordTypes.values().forEach(recordTypeBuilder -> recordTypeBuilder.buildDescriptor(fileBuilder));
final Descriptors.FileDescriptor fileDescriptor;
try {
final Descriptors.FileDescriptor[] dependencies = new Descriptors.FileDescriptor[] { unionDescriptor.getFile() };
fileDescriptor = Descriptors.FileDescriptor.buildFrom(fileBuilder.build(), dependencies);
} catch (Descriptors.DescriptorValidationException ex) {
throw new MetaDataException("Could not build synthesized file descriptor", ex);
}
for (SyntheticRecordTypeBuilder<?> recordTypeBuilder : syntheticRecordTypes.values()) {
builtSyntheticRecordTypes.put(recordTypeBuilder.getName(), recordTypeBuilder.build(metaData, fileDescriptor));
}
}
if (validate) {
final MetaDataValidator validator = new MetaDataValidator(metaData, indexMaintainerRegistry);
validator.validate();
}
return metaData;
}
use of com.apple.foundationdb.record.metadata.SyntheticRecordType in project fdb-record-layer by FoundationDB.
the class FDBRecordStore method loadSyntheticRecord.
/**
* Load a {@link FDBSyntheticRecord synthetic record} by loading its stored constituent records and synthesizing it from them.
* @param primaryKey the primary key of the synthetic record, which includes the primary keys of the constituents
* @return a future which completes to the synthesized record
*/
@Nonnull
@API(API.Status.EXPERIMENTAL)
public CompletableFuture<FDBSyntheticRecord> loadSyntheticRecord(@Nonnull Tuple primaryKey) {
SyntheticRecordType<?> syntheticRecordType = getRecordMetaData().getSyntheticRecordTypeFromRecordTypeKey(primaryKey.get(0));
int nconstituents = syntheticRecordType.getConstituents().size();
if (nconstituents != primaryKey.size() - 1) {
throw recordCoreException("Primary key does not have correct number of nested keys: " + primaryKey);
}
final Map<String, FDBStoredRecord<? extends Message>> constituents = new ConcurrentHashMap<>(nconstituents);
final CompletableFuture<?>[] futures = new CompletableFuture<?>[nconstituents];
for (int i = 0; i < nconstituents; i++) {
final SyntheticRecordType.Constituent constituent = syntheticRecordType.getConstituents().get(i);
final Tuple constituentKey = primaryKey.getNestedTuple(i + 1);
if (constituentKey == null) {
futures[i] = AsyncUtil.DONE;
} else {
futures[i] = loadRecordAsync(constituentKey).thenApply(rec -> {
if (rec == null) {
throw new RecordDoesNotExistException("constituent record not found: " + constituent.getName());
}
constituents.put(constituent.getName(), rec);
return null;
});
}
}
return CompletableFuture.allOf(futures).thenApply(vignore -> FDBSyntheticRecord.of(syntheticRecordType, constituents));
}
Aggregations