Search in sources :

Example 21 with RecordType

use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.

the class LuceneIndexTest method searchForAutoCompleteWithTextSizeLimit.

void searchForAutoCompleteWithTextSizeLimit(int limit, boolean matches) throws Exception {
    try (FDBRecordContext context = openContext(RecordLayerPropertyStorage.newBuilder().addProp(LuceneRecordContextProperties.LUCENE_AUTO_COMPLETE_TEXT_SIZE_UPPER_LIMIT, limit))) {
        final RecordType recordType = addIndexAndSaveRecordForAutoComplete(context, false);
        List<IndexEntry> results = recordStore.scanIndex(SIMPLE_TEXT_WITH_AUTO_COMPLETE, IndexScanType.BY_LUCENE_AUTO_COMPLETE, TupleRange.allOf(Tuple.from("software engineer")), null, ScanProperties.FORWARD_SCAN).asList().get();
        if (!matches) {
            // Assert no suggestions
            assertTrue(results.isEmpty());
            return;
        }
        // Assert the count of suggestions
        assertEquals(1, results.size());
        // Assert the suggestions' keys
        List<String> suggestions = results.stream().map(i -> (String) i.getKey().get(i.getKeySize() - 1)).collect(Collectors.toList());
        assertEquals(ImmutableList.of(ENGINEER_JOKE), suggestions);
        // Assert the corresponding field for the suggestions
        List<String> fields = results.stream().map(i -> (String) i.getKey().get(i.getKeySize() - 2)).collect(Collectors.toList());
        assertEquals(ImmutableList.of("text"), fields);
        results.stream().forEach(i -> assertDocumentPartialRecordFromIndexEntry(recordType, i, (String) i.getKey().get(i.getKeySize() - 1), (String) i.getKey().get(i.getKeySize() - 2), IndexScanType.BY_LUCENE_AUTO_COMPLETE));
        assertEquals(1, context.getTimer().getCounter(FDBStoreTimer.Counts.LUCENE_SCAN_MATCHED_AUTO_COMPLETE_SUGGESTIONS).getCount());
        assertEntriesAndSegmentInfoStoredInCompoundFile(AutoCompleteSuggesterCommitCheckAsync.getSuggestionIndexSubspace(recordStore.indexSubspace(SIMPLE_TEXT_WITH_AUTO_COMPLETE), TupleHelpers.EMPTY), context, "_0.cfs", true);
    }
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) Arrays(java.util.Arrays) StoreTimer(com.apple.foundationdb.record.provider.common.StoreTimer) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Random(java.util.Random) SynonymAnalyzer(com.apple.foundationdb.record.lucene.synonym.SynonymAnalyzer) EnglishSynonymMapConfig(com.apple.foundationdb.record.lucene.synonym.EnglishSynonymMapConfig) Subspace(com.apple.foundationdb.subspace.Subspace) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) Pair(org.apache.commons.lang3.tuple.Pair) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) COMPLEX_DOC(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.COMPLEX_DOC) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Tag(org.junit.jupiter.api.Tag) SynonymMapConfig(com.apple.foundationdb.record.lucene.synonym.SynonymMapConfig) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) Collectors(java.util.stream.Collectors) FDBRecordContextConfig(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContextConfig) IsCollectionWithSize.hasSize(org.hamcrest.collection.IsCollectionWithSize.hasSize) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) TestRecordsTextProto(com.apple.foundationdb.record.TestRecordsTextProto) List(java.util.List) NgramAnalyzer(com.apple.foundationdb.record.lucene.ngram.NgramAnalyzer) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) RecordCursorProto(com.apple.foundationdb.record.RecordCursorProto) FDBDirectory(com.apple.foundationdb.record.lucene.directory.FDBDirectory) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Expressions.function(com.apple.foundationdb.record.metadata.Key.Expressions.function) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) RecordLayerPropertyStorage(com.apple.foundationdb.record.provider.foundationdb.properties.RecordLayerPropertyStorage) Descriptors(com.google.protobuf.Descriptors) CompletableFuture(java.util.concurrent.CompletableFuture) ArrayList(java.util.ArrayList) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) IndexKeyValueToPartialRecord(com.apple.foundationdb.record.query.plan.IndexKeyValueToPartialRecord) ImmutableList(com.google.common.collect.ImmutableList) ScanProperties(com.apple.foundationdb.record.ScanProperties) FDBLuceneFileReference(com.apple.foundationdb.record.lucene.directory.FDBLuceneFileReference) TextIndexTestUtils(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) Nullable(javax.annotation.Nullable) SIMPLE_DOC(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.SIMPLE_DOC) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) Tags(com.apple.test.Tags) SequenceInputStream(java.io.SequenceInputStream) IndexFileNames(org.apache.lucene.index.IndexFileNames) IOException(java.io.IOException) FDBRecordStoreTestBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase) ExecutionException(java.util.concurrent.ExecutionException) RecordType(com.apple.foundationdb.record.metadata.RecordType) Index(com.apple.foundationdb.record.metadata.Index) AutoService(com.google.auto.service.AutoService) Message(com.google.protobuf.Message) Assertions(org.junit.jupiter.api.Assertions) RecordCursor(com.apple.foundationdb.record.RecordCursor) Collections(java.util.Collections) AllSuffixesTextTokenizer(com.apple.foundationdb.record.provider.common.text.AllSuffixesTextTokenizer) InputStream(java.io.InputStream) RecordType(com.apple.foundationdb.record.metadata.RecordType) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) IndexEntry(com.apple.foundationdb.record.IndexEntry)

Example 22 with RecordType

use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.

the class LuceneIndexQueryPlan method executePlan.

/**
 * Override here to have specific logic to build the {@link QueryResult} for lucene auto complete suggestion result.
 */
@Nonnull
@Override
public <M extends Message> RecordCursor<QueryResult> executePlan(@Nonnull FDBRecordStoreBase<M> store, @Nonnull EvaluationContext context, @Nullable byte[] continuation, @Nonnull ExecuteProperties executeProperties) {
    final RecordMetaData metaData = store.getRecordMetaData();
    final Index index = metaData.getIndex(indexName);
    final Collection<RecordType> recordTypes = metaData.recordTypesForIndex(index);
    if (recordTypes.size() != 1) {
        throw new RecordCoreException("No lucene index should span multiple record types");
    }
    final IndexScanType scanType = getScanType();
    if (scanType == IndexScanType.BY_LUCENE_AUTO_COMPLETE || scanType == IndexScanType.BY_LUCENE_SPELLCHECK) {
        final RecordType recordType = recordTypes.iterator().next();
        final RecordCursor<IndexEntry> entryRecordCursor = executeEntries(store, context, continuation, executeProperties);
        return entryRecordCursor.map(QueryPlanUtils.getCoveringIndexEntryToPartialRecordFunction(store, recordType.getName(), indexName, getToPartialRecord(index, recordType, scanType), scanType)).map(QueryResult::of);
    }
    return super.executePlan(store, context, continuation, executeProperties);
}
Also used : RecordMetaData(com.apple.foundationdb.record.RecordMetaData) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) QueryResult(com.apple.foundationdb.record.query.plan.plans.QueryResult) RecordType(com.apple.foundationdb.record.metadata.RecordType) IndexScanType(com.apple.foundationdb.record.IndexScanType) IndexEntry(com.apple.foundationdb.record.IndexEntry) Index(com.apple.foundationdb.record.metadata.Index) Nonnull(javax.annotation.Nonnull)

Example 23 with RecordType

use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.

the class LuceneIndexValidator method validate.

@Override
public void validate(@Nonnull MetaDataValidator metaDataValidator) {
    super.validate(metaDataValidator);
    validateNotVersion();
    for (RecordType recordType : metaDataValidator.getRecordMetaData().recordTypesForIndex(index)) {
        LuceneIndexExpressions.validate(index.getRootExpression(), recordType.getDescriptor());
    }
}
Also used : RecordType(com.apple.foundationdb.record.metadata.RecordType)

Example 24 with RecordType

use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.

the class RecordMetaDataBuilder method build.

/**
 * Build and validate meta-data with specific index registry.
 * @param validate {@code true} to validate the new meta-data
 * @return new meta-data
 */
@Nonnull
public RecordMetaData build(boolean validate) {
    Map<String, RecordType> builtRecordTypes = Maps.newHashMapWithExpectedSize(recordTypes.size());
    Map<String, SyntheticRecordType<?>> builtSyntheticRecordTypes = Maps.newHashMapWithExpectedSize(syntheticRecordTypes.size());
    RecordMetaData metaData = new RecordMetaData(recordsDescriptor, getUnionDescriptor(), unionFields, builtRecordTypes, builtSyntheticRecordTypes, indexes, universalIndexes, formerIndexes, splitLongRecords, storeRecordVersions, version, subspaceKeyCounter, usesSubspaceKeyCounter, recordCountKey, localFileDescriptor != null);
    for (RecordTypeBuilder recordTypeBuilder : recordTypes.values()) {
        KeyExpression primaryKey = recordTypeBuilder.getPrimaryKey();
        if (primaryKey != null) {
            builtRecordTypes.put(recordTypeBuilder.getName(), recordTypeBuilder.build(metaData));
            for (Index index : recordTypeBuilder.getIndexes()) {
                index.setPrimaryKeyComponentPositions(buildPrimaryKeyComponentPositions(index.getRootExpression(), primaryKey));
            }
        } else {
            throw new MetaDataException("Record type " + recordTypeBuilder.getName() + " must have a primary key");
        }
    }
    if (!syntheticRecordTypes.isEmpty()) {
        DescriptorProtos.FileDescriptorProto.Builder fileBuilder = DescriptorProtos.FileDescriptorProto.newBuilder();
        fileBuilder.setName("_synthetic");
        fileBuilder.addDependency(unionDescriptor.getFile().getName());
        syntheticRecordTypes.values().forEach(recordTypeBuilder -> recordTypeBuilder.buildDescriptor(fileBuilder));
        final Descriptors.FileDescriptor fileDescriptor;
        try {
            final Descriptors.FileDescriptor[] dependencies = new Descriptors.FileDescriptor[] { unionDescriptor.getFile() };
            fileDescriptor = Descriptors.FileDescriptor.buildFrom(fileBuilder.build(), dependencies);
        } catch (Descriptors.DescriptorValidationException ex) {
            throw new MetaDataException("Could not build synthesized file descriptor", ex);
        }
        for (SyntheticRecordTypeBuilder<?> recordTypeBuilder : syntheticRecordTypes.values()) {
            builtSyntheticRecordTypes.put(recordTypeBuilder.getName(), recordTypeBuilder.build(metaData, fileDescriptor));
        }
    }
    if (validate) {
        final MetaDataValidator validator = new MetaDataValidator(metaData, indexMaintainerRegistry);
        validator.validate();
    }
    return metaData;
}
Also used : SyntheticRecordType(com.apple.foundationdb.record.metadata.SyntheticRecordType) LiteralKeyExpression(com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression) FieldKeyExpression(com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) FormerIndex(com.apple.foundationdb.record.metadata.FormerIndex) Index(com.apple.foundationdb.record.metadata.Index) SyntheticRecordTypeBuilder(com.apple.foundationdb.record.metadata.SyntheticRecordTypeBuilder) RecordTypeBuilder(com.apple.foundationdb.record.metadata.RecordTypeBuilder) JoinedRecordTypeBuilder(com.apple.foundationdb.record.metadata.JoinedRecordTypeBuilder) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) RecordType(com.apple.foundationdb.record.metadata.RecordType) SyntheticRecordType(com.apple.foundationdb.record.metadata.SyntheticRecordType) Descriptors(com.google.protobuf.Descriptors) MetaDataValidator(com.apple.foundationdb.record.metadata.MetaDataValidator) Nonnull(javax.annotation.Nonnull)

Example 25 with RecordType

use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method repairRecordKeyIfNecessary.

private void repairRecordKeyIfNecessary(@Nonnull FDBRecordContext context, @Nonnull Subspace recordSubspace, @Nonnull KeyValue keyValue, final boolean isDryRun) {
    final RecordMetaData metaData = metaDataProvider.getRecordMetaData();
    final Tuple recordKey = recordSubspace.unpack(keyValue.getKey());
    // Ignore version key
    if (metaData.isStoreRecordVersions() && isMaybeVersion(recordKey)) {
        return;
    }
    final Message protoRecord = serializer.deserialize(metaData, recordKey, keyValue.getValue(), getTimer());
    final RecordType recordType = metaData.getRecordTypeForDescriptor(protoRecord.getDescriptorForType());
    final KeyExpression primaryKeyExpression = recordType.getPrimaryKey();
    if (recordKey.size() == primaryKeyExpression.getColumnSize()) {
        context.increment(FDBStoreTimer.Counts.REPAIR_RECORD_KEY);
        final Tuple newPrimaryKey = recordKey.add(SplitHelper.UNSPLIT_RECORD);
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(KeyValueLogMessage.of("Repairing primary key", LogMessageKeys.RECORD_TYPE, recordType.getName(), subspaceProvider.logKey(), subspaceProvider.toString(context), "dry_run", isDryRun, "orig_primary_key", recordKey, "new_primary_key", newPrimaryKey));
        }
        if (!isDryRun) {
            final Transaction tr = context.ensureActive();
            tr.clear(keyValue.getKey());
            tr.set(recordSubspace.pack(newPrimaryKey), keyValue.getValue());
        }
    } else if (recordKey.size() == primaryKeyExpression.getColumnSize() + 1) {
        Object suffix = recordKey.get(recordKey.size() - 1);
        if (!(suffix instanceof Long) || !(((Long) suffix) == SplitHelper.UNSPLIT_RECORD)) {
            context.increment(FDBStoreTimer.Counts.INVALID_SPLIT_SUFFIX);
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug(KeyValueLogMessage.of("Invalid split suffix", subspaceProvider.logKey(), subspaceProvider.toString(context), LogMessageKeys.RECORD_TYPE, recordType.getName(), LogMessageKeys.PRIMARY_KEY, recordKey));
            }
        }
    } else {
        context.increment(FDBStoreTimer.Counts.INVALID_KEY_LENGTH);
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(KeyValueLogMessage.of("Invalid key length", subspaceProvider.logKey(), subspaceProvider.toString(context), LogMessageKeys.RECORD_TYPE, recordType.getName(), LogMessageKeys.PRIMARY_KEY, recordKey));
        }
    }
}
Also used : RecordMetaData(com.apple.foundationdb.record.RecordMetaData) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Message(com.google.protobuf.Message) SyntheticRecordType(com.apple.foundationdb.record.metadata.SyntheticRecordType) RecordType(com.apple.foundationdb.record.metadata.RecordType) Transaction(com.apple.foundationdb.Transaction) ReadTransaction(com.apple.foundationdb.ReadTransaction) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) EmptyKeyExpression(com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression) AtomicLong(java.util.concurrent.atomic.AtomicLong) Tuple(com.apple.foundationdb.tuple.Tuple)

Aggregations

RecordType (com.apple.foundationdb.record.metadata.RecordType)43 Nonnull (javax.annotation.Nonnull)29 Index (com.apple.foundationdb.record.metadata.Index)25 KeyExpression (com.apple.foundationdb.record.metadata.expressions.KeyExpression)24 RecordMetaData (com.apple.foundationdb.record.RecordMetaData)20 Nullable (javax.annotation.Nullable)20 Descriptors (com.google.protobuf.Descriptors)18 SyntheticRecordType (com.apple.foundationdb.record.metadata.SyntheticRecordType)17 ArrayList (java.util.ArrayList)17 RecordCoreException (com.apple.foundationdb.record.RecordCoreException)16 Tuple (com.apple.foundationdb.tuple.Tuple)16 List (java.util.List)15 Collection (java.util.Collection)14 API (com.apple.foundationdb.annotation.API)13 IndexEntry (com.apple.foundationdb.record.IndexEntry)13 Collectors (java.util.stream.Collectors)13 RecordCoreArgumentException (com.apple.foundationdb.record.RecordCoreArgumentException)12 TupleRange (com.apple.foundationdb.record.TupleRange)12 MetaDataException (com.apple.foundationdb.record.metadata.MetaDataException)12 CompletableFuture (java.util.concurrent.CompletableFuture)12