Examples with RecordType com.apple.foundationdb.record.metadata.RecordType used on opensource projects

Example 36 with RecordType

use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.

the class StandardIndexMaintainer method validateMissingEntries.

/**
 * Validate entries in the index. It scans the records and checks if the index entries associated with each record
 * exist. Note that it may not work for indexes on synthetic record types (e.g., join indexes).
 * @param continuation any continuation from a previous validation invocation
 * @param scanProperties skip, limit and other properties of the validation
 * @return a cursor over records that have no associated index entries including the reason
 */
@Nonnull
protected RecordCursor<InvalidIndexEntry> validateMissingEntries(@Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    final Collection<RecordType> recordTypes = state.store.getRecordMetaData().recordTypesForIndex(state.index);
    final FDBRecordStoreBase.PipelineSizer pipelineSizer = state.store.getPipelineSizer();
    return RecordCursor.flatMapPipelined(cont -> state.store.scanRecords(TupleRange.ALL, cont, scanProperties).filter(rec -> recordTypes.contains(rec.getRecordType())), (record, cont) -> {
        List<IndexEntry> filteredIndexEntries = filteredIndexEntries(record);
        return RecordCursor.fromList(filteredIndexEntries == null ? Collections.emptyList() : filteredIndexEntries.stream().map(indexEntryWithoutPrimaryKey -> new IndexEntry(indexEntryWithoutPrimaryKey.getIndex(), indexEntryKey(indexEntryWithoutPrimaryKey.getKey(), record.getPrimaryKey()), indexEntryWithoutPrimaryKey.getValue())).map(indexEntry -> Pair.of(indexEntry, record)).collect(Collectors.toList()), cont);
    }, continuation, pipelineSizer.getPipelineSize(PipelineOperation.RECORD_FUNCTION)).filterAsync(indexEntryRecordPair -> {
        final byte[] keyBytes = state.indexSubspace.pack(indexEntryRecordPair.getLeft().getKey());
        return state.transaction.get(keyBytes).thenApply(Objects::isNull);
    }, pipelineSizer.getPipelineSize(PipelineOperation.INDEX_ASYNC_FILTER)).map(indexEntryKeyRecordPair -> InvalidIndexEntry.newMissing(indexEntryKeyRecordPair.getLeft(), indexEntryKeyRecordPair.getRight()));
}

Example 37 with RecordType

use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.

the class KeySpaceCountTree method resolveNonDirectory.

/**
 * Resolve something other than a {@link KeySpaceDirectory} node.
 * @param context an open transaction to use to read from the database
 * @param resolvedParent the resolved parent node
 * @param object the {@link com.apple.foundationdb.tuple.Tuple} element for this node
 * @return a future that completes to a new {@link Resolved} or {@code null}
 */
protected CompletableFuture<Resolved> resolveNonDirectory(@Nonnull FDBRecordContext context, @Nonnull Resolved resolvedParent, @Nullable Object object) {
    int distance = 0;
    ResolvedRecordStoreKeyspace recordStoreKeyspace = null;
    ResolvedRecordTypeKeyspace recordTypeKeyspace = null;
    ResolvedIndexKeyspace indexKeyspace = null;
    for (Resolved resolved = resolvedParent; resolved != null && resolved.getDirectory() == null; resolved = resolved.getParent()) {
        if (resolved instanceof ResolvedRecordTypeKeyspace) {
            recordTypeKeyspace = (ResolvedRecordTypeKeyspace) resolved;
            break;
        }
        if (resolved instanceof ResolvedIndexKeyspace) {
            indexKeyspace = (ResolvedIndexKeyspace) resolved;
            break;
        }
        if (resolved instanceof ResolvedRecordStoreKeyspace) {
            recordStoreKeyspace = (ResolvedRecordStoreKeyspace) resolved;
            break;
        }
        distance++;
    }
    if (recordStoreKeyspace != null && recordStoreKeyspace.getRecordMetaData() != null) {
        switch(recordStoreKeyspace.getRecordStoreKeyspace()) {
            case RECORD:
                if (distance == 0 && object != null && recordStoreKeyspace.getRecordMetaData().primaryKeyHasRecordTypePrefix()) {
                    final RecordType recordType;
                    try {
                        recordType = recordStoreKeyspace.getRecordMetaData().getRecordTypeFromRecordTypeKey(object);
                    } catch (RecordCoreException ex) {
                        break;
                    }
                    return CompletableFuture.completedFuture(new ResolvedRecordTypeKeyspace(resolvedParent, recordType));
                }
                KeyExpression commonPrimaryKey = recordStoreKeyspace.getRecordMetaData().commonPrimaryKey();
                if (commonPrimaryKey != null) {
                    List<KeyExpression> storedPrimaryKeys = commonPrimaryKey.normalizeKeyForPositions();
                    if (distance < storedPrimaryKeys.size()) {
                        return resolvePrimaryKeyField(context, resolvedParent, object, storedPrimaryKeys.get(distance), distance);
                    }
                }
                break;
            case INDEX:
            case INDEX_SECONDARY_SPACE:
            case INDEX_RANGE_SPACE:
            case INDEX_UNIQUENESS_VIOLATIONS_SPACE:
            case INDEX_BUILD_SPACE:
                // Once https://github.com/FoundationDB/fdb-record-layer/issues/514 is addressed, that will need this, too.
                if (distance == 0 && object != null) {
                    final Index index;
                    try {
                        index = recordStoreKeyspace.getRecordMetaData().getIndexFromSubspaceKey(object);
                    } catch (RecordCoreException ex) {
                        break;
                    }
                    return CompletableFuture.completedFuture(new ResolvedIndexKeyspace(resolvedParent, index));
                }
                break;
            default:
                break;
        }
    }
    if (recordTypeKeyspace != null) {
        List<KeyExpression> storedPrimaryKeys = recordTypeKeyspace.getRecordType().getPrimaryKey().normalizeKeyForPositions();
        if (distance + 1 < storedPrimaryKeys.size()) {
            return resolvePrimaryKeyField(context, resolvedParent, object, storedPrimaryKeys.get(distance + 1), distance + 1);
        }
    }
    if (indexKeyspace != null && indexKeyspace.getParent() instanceof ResolvedRecordStoreKeyspace && ((ResolvedRecordStoreKeyspace) indexKeyspace.getParent()).getRecordStoreKeyspace() == FDBRecordStoreKeyspace.INDEX) {
        Index index = indexKeyspace.getIndex();
        List<KeyExpression> storedKeys = indexStoredKeys(index);
        if (distance < storedKeys.size()) {
            return resolveIndexField(context, resolvedParent, object, index, storedKeys.get(distance), distance);
        }
    }
    return UNRESOLVED;
}

Example 38 with RecordType

use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexMaintainerFactory method getIndexValidator.

@Override
@Nonnull
public IndexValidator getIndexValidator(Index index) {
    return new IndexValidator(index) {

        @Override
        public void validate(@Nonnull MetaDataValidator metaDataValidator) {
            super.validate(metaDataValidator);
            validateGrouping(1);
            final GroupingKeyExpression group = (GroupingKeyExpression) index.getRootExpression();
            if (group.getGroupedCount() != 1) {
                throw new KeyExpression.InvalidExpressionException(String.format("%s index needs grouped position", index.getType()), LogMessageKeys.INDEX_NAME, index.getName(), LogMessageKeys.INDEX_KEY, index.getRootExpression());
            }
            validateNotVersion();
        }

        @Override
        @SuppressWarnings("fallthrough")
        public void validateIndexForRecordType(@Nonnull RecordType recordType, @Nonnull MetaDataValidator metaDataValidator) {
            final List<Descriptors.FieldDescriptor> fields = metaDataValidator.validateIndexForRecordType(index, recordType);
            switch(fields.get(fields.size() - 1).getType()) {
                case INT64:
                case UINT64:
                case INT32:
                case UINT32:
                case SINT32:
                case SINT64:
                case FIXED32:
                case FIXED64:
                case SFIXED32:
                case SFIXED64:
                    break;
                default:
                    throw new KeyExpression.InvalidExpressionException(String.format("%s index only supports integer position key", index.getType()), LogMessageKeys.INDEX_NAME, index.getName(), LogMessageKeys.INDEX_KEY, index.getRootExpression(), "record_type", recordType.getName());
            }
        }
    };
}

Example 39 with RecordType

use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.

the class TextIndexMaintainerFactory method getIndexValidator.

/**
 * Validates that the index provided is valid for text indexes. This means that
 * the index must:
 *
 * <ul>
 *     <li>Not be a unique index.</li>
 *     <li>Not include a {@link com.apple.foundationdb.record.metadata.expressions.VersionKeyExpression#VERSION} expression in its root expression.</li>
 *     <li>Have a key expression whose first column is of type <code>string</code> (possibly with grouping columns
 *         before the tokenized text column) and is not repeated.</li> <!--Maybe we should support FanType.Concatenate?-->
 *     <li>Specify a valid tokenizer and tokenizer version through the index options (possibly using the defaults).</li>
 *     <li>Not define a value expression.</li>
 * </ul>
 *
 * @param index the index to validate
 * @return a validator to run against the index
 * @throws KeyExpression.InvalidExpressionException if the expression does not contain a string as its first ungrouped column
 * @throws com.apple.foundationdb.record.metadata.MetaDataException if the tokenizer is not defined, if the tokenizer version
 *                                                    is out of range, or if the index is marked as unique
 */
@Nonnull
@Override
public IndexValidator getIndexValidator(Index index) {
    return new IndexValidator(index) {

        @Override
        public void validate(@Nonnull MetaDataValidator metaDataValidator) {
            super.validate(metaDataValidator);
            validateNotVersion();
            validateNotUnique();
            // TODO: allow value expressions for covering text indexes
            validateNoValue();
            // Validate that the tokenizer exists and that the version is in a valid range.
            TextTokenizer tokenizer = TextIndexMaintainer.getTokenizer(index);
            int tokenizerVersion = TextIndexMaintainer.getIndexTokenizerVersion(index);
            tokenizer.validateVersion(tokenizerVersion);
        }

        @Override
        public void validateIndexForRecordType(@Nonnull RecordType recordType, @Nonnull MetaDataValidator metaDataValidator) {
            final List<Descriptors.FieldDescriptor> fields = metaDataValidator.validateIndexForRecordType(index, recordType);
            int textFieldPosition = TextIndexMaintainer.textFieldPosition(index.getRootExpression());
            if (textFieldPosition > fields.size()) {
                throw new KeyExpression.InvalidExpressionException("text index does not have text field after grouped fields");
            } else {
                Descriptors.FieldDescriptor textFieldDescriptor = fields.get(textFieldPosition);
                if (!textFieldDescriptor.getType().equals(Descriptors.FieldDescriptor.Type.STRING)) {
                    throw new KeyExpression.InvalidExpressionException(String.format("text index has non-string type %s as text field", textFieldDescriptor.getLiteJavaType()));
                }
                if (textFieldDescriptor.isRepeated()) {
                    throw new KeyExpression.InvalidExpressionException("text index does not allow a repeated field for text body");
                }
            }
        }

        /**
         * Validate any options that have changed. There are several options unique to text indexes which
         * may change without requiring the index be rebuilt. They are:
         *
         * <ul>
         *     <li>{@link IndexOptions#TEXT_TOKENIZER_VERSION_OPTION} which can be increased (but not decreased)</li>
         *     <li>{@link IndexOptions#TEXT_ADD_AGGRESSIVE_CONFLICT_RANGES_OPTION} which only affects what conflict ranges
         *          are added at index update time and thus has no impact on the on-disk representation</li>
         *     <li>{@link IndexOptions#TEXT_OMIT_POSITIONS_OPTION} which changes whether the position lists are included
         *          in index entries</li>
         * </ul>
         *
         * <p>
         * Note that the {@link IndexOptions#TEXT_TOKENIZER_NAME_OPTION} is <em>not</em> allowed to change
         * (without rebuilding the index).
         * </p>
         *
         * @param oldIndex an older version of this index
         * @param changedOptions the set of changed options
         */
        @Override
        protected void validateChangedOptions(@Nonnull Index oldIndex, @Nonnull Set<String> changedOptions) {
            for (String changedOption : changedOptions) {
                switch(changedOption) {
                    case IndexOptions.TEXT_ADD_AGGRESSIVE_CONFLICT_RANGES_OPTION:
                    case IndexOptions.TEXT_OMIT_POSITIONS_OPTION:
                        // without breaking compatibility.
                        break;
                    case IndexOptions.TEXT_TOKENIZER_NAME_OPTION:
                        String oldTokenizerName = TextIndexMaintainer.getTokenizer(oldIndex).getName();
                        String newTokenizerName = TextIndexMaintainer.getTokenizer(index).getName();
                        if (!oldTokenizerName.equals(newTokenizerName)) {
                            throw new MetaDataException("text tokenizer changed", LogMessageKeys.INDEX_NAME, index.getName());
                        }
                        break;
                    case IndexOptions.TEXT_TOKENIZER_VERSION_OPTION:
                        // The tokenizer version should always go up.
                        int oldTokenizerVersion = TextIndexMaintainer.getIndexTokenizerVersion(oldIndex);
                        int newTokenizerVersion = TextIndexMaintainer.getIndexTokenizerVersion(index);
                        if (oldTokenizerVersion > newTokenizerVersion) {
                            throw new MetaDataException("text tokenizer version downgraded", LogMessageKeys.INDEX_NAME, index.getName(), LogMessageKeys.OLD_VERSION, oldTokenizerVersion, LogMessageKeys.NEW_VERSION, newTokenizerVersion);
                        }
                        break;
                    default:
                        // Changed options that are not text options will be handled by super class
                        if (TEXT_OPTIONS.contains(changedOption)) {
                            throw new MetaDataException("index option changed", LogMessageKeys.INDEX_NAME, index.getName(), LogMessageKeys.INDEX_OPTION, changedOption, LogMessageKeys.OLD_OPTION, oldIndex.getOption(changedOption), LogMessageKeys.NEW_OPTION, index.getOption(changedOption));
                        }
                }
            }
            changedOptions.removeAll(TEXT_OPTIONS);
            super.validateChangedOptions(oldIndex, changedOptions);
        }
    };
}

Example 40 with RecordType

use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.

the class SortedRecordSerializer method deserialize.

@Nonnull
public FDBQueriedRecord<M> deserialize(@Nonnull RecordSortingProto.SortedRecord sortedRecord) {
    final byte[] primaryKeyBytes = sortedRecord.getPrimaryKey().toByteArray();
    final Tuple primaryKey = Tuple.fromBytes(primaryKeyBytes);
    final byte[] recordBytes = sortedRecord.getMessage().toByteArray();
    final M record = serializer.deserialize(recordMetaData, primaryKey, recordBytes, timer);
    final RecordType recordType = recordMetaData.getRecordTypeForDescriptor(record.getDescriptorForType());
    final FDBRecordVersion version;
    if (sortedRecord.hasVersion()) {
        version = FDBRecordVersion.fromBytes(sortedRecord.getVersion().toByteArray());
    } else {
        version = null;
    }
    return new Sorted<>(primaryKey, recordType, record, version);
}