Examples with MetaDataException com.apple.foundationdb.record.metadata.MetaDataException used on opensource projects

Example 51 with MetaDataException

use of com.apple.foundationdb.record.metadata.MetaDataException in project fdb-record-layer by FoundationDB.

the class RecordMetaDataBuilder method validateUnion.

private static void validateUnion(@Nonnull Descriptors.FileDescriptor fileDescriptor, @Nonnull Descriptors.Descriptor unionDescriptor) {
    for (Descriptors.FieldDescriptor unionField : unionDescriptor.getFields()) {
        // Only message types allowed.
        if (unionField.getType() != Descriptors.FieldDescriptor.Type.MESSAGE) {
            throw new MetaDataException("Union field " + unionField.getName() + " is not a message");
        }
        // No repeating fields.
        if (unionField.isRepeated()) {
            throw new MetaDataException("Union field " + unionField.getName() + " should not be repeated");
        }
        Descriptors.Descriptor descriptor = unionField.getMessageType();
        // RecordTypeUnion is reserved for union descriptor and cannot appear as a union fields
        if (DEFAULT_UNION_NAME.equals(descriptor.getName())) {
            throw new MetaDataException("Union message type " + descriptor.getName() + " cannot be a union field.");
        }
        // All union fields must be RECORD (i.e., they cannot be NESTED/UNIONs). The same rule applies to imported union fields too.
        RecordMetaDataOptionsProto.RecordTypeOptions recordTypeOptions = descriptor.getOptions().getExtension(RecordMetaDataOptionsProto.record);
        if (recordTypeOptions != null && recordTypeOptions.hasUsage() && recordTypeOptions.getUsage() != RecordMetaDataOptionsProto.RecordTypeOptions.Usage.RECORD) {
            throw new MetaDataException("Union field " + unionField.getName() + " has type " + descriptor.getName() + " which is not a record");
        }
    }
    // All RECORD message types defined in this proto must be present in the union.
    for (Descriptors.Descriptor descriptor : fileDescriptor.getMessageTypes()) {
        RecordMetaDataOptionsProto.RecordTypeOptions recordTypeOptions = descriptor.getOptions().getExtension(RecordMetaDataOptionsProto.record);
        if (recordTypeOptions != null && recordTypeOptions.hasUsage()) {
            switch(recordTypeOptions.getUsage()) {
                case RECORD:
                    if (DEFAULT_UNION_NAME.equals(descriptor.getName())) {
                        if (unionHasMessageType(unionDescriptor, descriptor)) {
                            throw new MetaDataException("Union message type " + descriptor.getName() + " cannot be a union field.");
                        }
                    } else if (!unionHasMessageType(unionDescriptor, descriptor)) {
                        throw new MetaDataException("Record message type " + descriptor.getName() + " must be a union field.");
                    }
                    break;
                case UNION:
                // Already checked above that none of the union fields is UNION.
                case NESTED:
                // Already checked above that none of the union fields is NESTED.
                case UNSET:
                // is that the message type's name might equal DEFAULT_UNION_NAME. This is already checked above.
                default:
                    break;
            }
        }
    }
}

Example 52 with MetaDataException

use of com.apple.foundationdb.record.metadata.MetaDataException in project fdb-record-layer by FoundationDB.

the class RecordMetaDataBuilder method protoFieldOptions.

@SuppressWarnings("deprecation")
private void protoFieldOptions(RecordTypeBuilder recordType, Descriptors.FieldDescriptor fieldDescriptor, RecordMetaDataOptionsProto.FieldOptions fieldOptions) {
    Descriptors.Descriptor descriptor = recordType.getDescriptor();
    if (fieldOptions.hasIndex() || fieldOptions.hasIndexed()) {
        String type;
        Map<String, String> options;
        if (fieldOptions.hasIndex()) {
            RecordMetaDataOptionsProto.FieldOptions.IndexOption indexOption = fieldOptions.getIndex();
            type = indexOption.getType();
            options = Index.buildOptions(indexOption.getOptionsList(), indexOption.getUnique());
        } else {
            type = Index.indexTypeToType(fieldOptions.getIndexed());
            options = Index.indexTypeToOptions(fieldOptions.getIndexed());
        }
        final FieldKeyExpression field = Key.Expressions.fromDescriptor(fieldDescriptor);
        final KeyExpression expr;
        if (type.equals(IndexTypes.RANK)) {
            expr = field.ungrouped();
        } else {
            expr = field;
        }
        final Index index = new Index(descriptor.getName() + "$" + fieldDescriptor.getName(), expr, Index.EMPTY_VALUE, type, options);
        addIndex(recordType, index);
    } else if (fieldOptions.getPrimaryKey()) {
        if (recordType.getPrimaryKey() != null) {
            throw new MetaDataException("Only one primary key per record type is allowed have: " + recordType.getPrimaryKey() + "; adding on " + fieldDescriptor.getName());
        } else {
            if (fieldDescriptor.isRepeated()) {
                // TODO maybe default to concatenate for this.
                throw new MetaDataException("Primary key cannot be set on a repeated field");
            } else {
                recordType.setPrimaryKey(Key.Expressions.fromDescriptor(fieldDescriptor));
            }
        }
    }
}

Example 53 with MetaDataException

use of com.apple.foundationdb.record.metadata.MetaDataException in project fdb-record-layer by FoundationDB.

the class RecordMetaDataBuilder method build.

/**
 * Build and validate meta-data with specific index registry.
 * @param validate {@code true} to validate the new meta-data
 * @return new meta-data
 */
@Nonnull
public RecordMetaData build(boolean validate) {
    Map<String, RecordType> builtRecordTypes = Maps.newHashMapWithExpectedSize(recordTypes.size());
    Map<String, SyntheticRecordType<?>> builtSyntheticRecordTypes = Maps.newHashMapWithExpectedSize(syntheticRecordTypes.size());
    RecordMetaData metaData = new RecordMetaData(recordsDescriptor, getUnionDescriptor(), unionFields, builtRecordTypes, builtSyntheticRecordTypes, indexes, universalIndexes, formerIndexes, splitLongRecords, storeRecordVersions, version, subspaceKeyCounter, usesSubspaceKeyCounter, recordCountKey, localFileDescriptor != null);
    for (RecordTypeBuilder recordTypeBuilder : recordTypes.values()) {
        KeyExpression primaryKey = recordTypeBuilder.getPrimaryKey();
        if (primaryKey != null) {
            builtRecordTypes.put(recordTypeBuilder.getName(), recordTypeBuilder.build(metaData));
            for (Index index : recordTypeBuilder.getIndexes()) {
                index.setPrimaryKeyComponentPositions(buildPrimaryKeyComponentPositions(index.getRootExpression(), primaryKey));
            }
        } else {
            throw new MetaDataException("Record type " + recordTypeBuilder.getName() + " must have a primary key");
        }
    }
    if (!syntheticRecordTypes.isEmpty()) {
        DescriptorProtos.FileDescriptorProto.Builder fileBuilder = DescriptorProtos.FileDescriptorProto.newBuilder();
        fileBuilder.setName("_synthetic");
        fileBuilder.addDependency(unionDescriptor.getFile().getName());
        syntheticRecordTypes.values().forEach(recordTypeBuilder -> recordTypeBuilder.buildDescriptor(fileBuilder));
        final Descriptors.FileDescriptor fileDescriptor;
        try {
            final Descriptors.FileDescriptor[] dependencies = new Descriptors.FileDescriptor[] { unionDescriptor.getFile() };
            fileDescriptor = Descriptors.FileDescriptor.buildFrom(fileBuilder.build(), dependencies);
        } catch (Descriptors.DescriptorValidationException ex) {
            throw new MetaDataException("Could not build synthesized file descriptor", ex);
        }
        for (SyntheticRecordTypeBuilder<?> recordTypeBuilder : syntheticRecordTypes.values()) {
            builtSyntheticRecordTypes.put(recordTypeBuilder.getName(), recordTypeBuilder.build(metaData, fileDescriptor));
        }
    }
    if (validate) {
        final MetaDataValidator validator = new MetaDataValidator(metaData, indexMaintainerRegistry);
        validator.validate();
    }
    return metaData;
}

Example 54 with MetaDataException

use of com.apple.foundationdb.record.metadata.MetaDataException in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method firstUnbuiltRange.

/**
 * Returns the first unbuilt range of an index that is currently being bulit.
 * If there is no range that is currently unbuilt, it will return an
 * empty {@link Optional}. If there is one, it will return an {@link Optional}
 * set to the first unbuilt range it finds.
 * @param index the index to check built state
 * @return a future that will contain the first unbuilt range if any
 */
@Nonnull
public CompletableFuture<Optional<Range>> firstUnbuiltRange(@Nonnull Index index) {
    if (!getRecordMetaData().hasIndex(index.getName())) {
        throw new MetaDataException("Index " + index.getName() + " does not exist in meta-data.");
    }
    Transaction tr = ensureContextActive();
    RangeSet rangeSet = new RangeSet(indexRangeSubspace(index));
    AsyncIterator<Range> missingRangeIterator = rangeSet.missingRanges(tr, null, null, 1).iterator();
    return missingRangeIterator.onHasNext().thenApply(hasFirst -> {
        if (hasFirst) {
            return Optional.of(missingRangeIterator.next());
        } else {
            return Optional.empty();
        }
    });
}

Example 55 with MetaDataException

use of com.apple.foundationdb.record.metadata.MetaDataException in project fdb-record-layer by FoundationDB.

the class IndexingScrubDangling method scrubIndexRangeOnly.

@Nonnull
private CompletableFuture<Boolean> scrubIndexRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    Index index = common.getIndex();
    final RecordMetaData metaData = store.getRecordMetaData();
    final RecordMetaDataProvider recordMetaDataProvider = common.getRecordStoreBuilder().getMetaDataProvider();
    if (recordMetaDataProvider == null || !metaData.equals(recordMetaDataProvider.getRecordMetaData())) {
        throw new MetaDataException("Store does not have the same metadata");
    }
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // scrubbing only readable, VALUE, idempotence indexes (at least for now)
    validateOrThrowEx(maintainer.isIdempotent(), "scrubbed index is not idempotent");
    validateOrThrowEx(index.getType().equals(IndexTypes.VALUE) || scrubbingPolicy.ignoreIndexTypeCheck(), "scrubbed index is not a VALUE index");
    validateOrThrowEx(store.getIndexState(index) == IndexState.READABLE, "scrubbed index is not readable");
    RangeSet rangeSet = new RangeSet(indexScrubIndexRangeSubspace(store, index));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// always respectLimit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // Here: no more missing ranges - all done
            // To avoid stale metadata, we'll keep the scrubbed-ranges indicator empty until the next scrub call.
            Transaction tr = store.getContext().ensureActive();
            tr.clear(indexScrubIndexRangeSubspace(store, index).range());
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        RecordCursor<FDBIndexedRecord<Message>> cursor = store.scanIndexRecords(index.getName(), IndexScanType.BY_VALUE, tupleRange, null, IndexOrphanBehavior.RETURN, scanProperties);
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        final AtomicReference<RecordCursorResult<FDBIndexedRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final long scanLimit = scrubbingPolicy.getEntriesScanLimit();
        return iterateRangeOnly(store, cursor, this::deleteIndexIfDangling, lastResult, hasMore, recordsScanned, true).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getIndexEntry().getKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> {
            if (scanLimit > 0) {
                scanCounter += recordsScanned.get();
                if (scanLimit <= scanCounter) {
                    return false;
                }
            }
            return !Objects.equals(cont, rangeEnd);
        }));
    });
}