use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.
the class RecordQueryPlannerSubstitutionVisitor method removeIndexFetch.
@Nullable
public static RecordQueryPlan removeIndexFetch(@Nonnull RecordMetaData recordMetaData, @Nonnull PlannableIndexTypes indexTypes, @Nullable KeyExpression commonPrimaryKey, @Nonnull RecordQueryPlan plan, @Nonnull Set<KeyExpression> requiredFields) {
if (plan instanceof RecordQueryPlanWithIndex) {
RecordQueryPlanWithIndex indexPlan = (RecordQueryPlanWithIndex) plan;
Index index = recordMetaData.getIndex(indexPlan.getIndexName());
final Collection<RecordType> recordTypes = recordMetaData.recordTypesForIndex(index);
if (recordTypes.size() != 1) {
return null;
}
final RecordType recordType = Iterables.getOnlyElement(recordTypes);
AvailableFields fieldsFromIndex = AvailableFields.fromIndex(recordType, index, indexTypes, commonPrimaryKey);
Set<KeyExpression> fields = new HashSet<>(requiredFields);
if (commonPrimaryKey != null) {
// Need the primary key, even if it wasn't one of the explicit result fields.
fields.addAll(commonPrimaryKey.normalizeKeyForPositions());
}
if (fieldsFromIndex.containsAll(fields)) {
final IndexKeyValueToPartialRecord keyValueToPartialRecord = fieldsFromIndex.buildIndexKeyValueToPartialRecord(recordType).build();
if (keyValueToPartialRecord != null) {
return new RecordQueryCoveringIndexPlan(indexPlan, recordType.getName(), fieldsFromIndex, keyValueToPartialRecord);
}
}
} else if (plan instanceof RecordQueryFetchFromPartialRecordPlan) {
RecordQueryFetchFromPartialRecordPlan fetchPlan = (RecordQueryFetchFromPartialRecordPlan) plan;
if (fetchPlan.getChild().getAvailableFields().containsAll(requiredFields)) {
return ((RecordQueryFetchFromPartialRecordPlan) plan).getChild();
}
}
return null;
}
use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.
the class LuceneIndexTest method searchForAutoCompleteAndAssert.
private void searchForAutoCompleteAndAssert(String query, boolean matches, boolean highlight, int textSizeLimit) throws Exception {
try (FDBRecordContext context = openContext(RecordLayerPropertyStorage.newBuilder().addProp(LuceneRecordContextProperties.LUCENE_AUTO_COMPLETE_TEXT_SIZE_UPPER_LIMIT, textSizeLimit))) {
final RecordType recordType = addIndexAndSaveRecordForAutoComplete(context, highlight);
List<IndexEntry> results = recordStore.scanIndex(highlight ? SIMPLE_TEXT_WITH_AUTO_COMPLETE_WITH_HIGHLIGHT : SIMPLE_TEXT_WITH_AUTO_COMPLETE, IndexScanType.BY_LUCENE_AUTO_COMPLETE, TupleRange.allOf(Tuple.from(query)), null, ScanProperties.FORWARD_SCAN).asList().get();
if (!matches) {
// Assert no suggestions
assertTrue(results.isEmpty());
assertEquals(0, context.getTimer().getCounter(FDBStoreTimer.Counts.LUCENE_SCAN_MATCHED_AUTO_COMPLETE_SUGGESTIONS).getCount());
return;
}
// Assert the count of suggestions
assertEquals(6, results.size());
// Assert the suggestions' keys
List<String> suggestions = results.stream().map(i -> (String) i.getKey().get(i.getKeySize() - 1)).collect(Collectors.toList());
if (highlight) {
assertEquals(ImmutableList.of("<b>good</b> evening", "<b>Good</b> night", "<b>Good</b> morning", "<b>Good</b> afternoon", "I'm <b>good</b>", "That's really <b>good</b>!"), suggestions);
} else {
assertEquals(ImmutableList.of("good evening", "Good night", "Good morning", "Good afternoon", "I'm good", "That's really good!"), suggestions);
}
// Assert the corresponding field for the suggestions
List<String> fields = results.stream().map(i -> (String) i.getKey().get(i.getKeySize() - 2)).collect(Collectors.toList());
assertEquals(ImmutableList.of("text", "text", "text", "text", "text", "text"), fields);
// Assert the suggestions are sorted according to their values, which are determined by the position of the term into the indexed text
List<Long> values = results.stream().map(i -> (Long) i.getValue().get(0)).collect(Collectors.toList());
List<Long> valuesSorted = new ArrayList<>(values);
Collections.sort(valuesSorted, Collections.reverseOrder());
assertEquals(valuesSorted, values);
assertEquals(values.get(0), values.get(1));
assertEquals(values.get(1), values.get(2));
assertEquals(values.get(2), values.get(3));
assertTrue(values.get(3) > values.get(4));
assertTrue(values.get(4) > values.get(5));
results.stream().forEach(i -> assertDocumentPartialRecordFromIndexEntry(recordType, i, (String) i.getKey().get(i.getKeySize() - 1), (String) i.getKey().get(i.getKeySize() - 2), IndexScanType.BY_LUCENE_AUTO_COMPLETE));
assertEquals(6, context.getTimer().getCounter(FDBStoreTimer.Counts.LUCENE_SCAN_MATCHED_AUTO_COMPLETE_SUGGESTIONS).getCount());
assertEntriesAndSegmentInfoStoredInCompoundFile(AutoCompleteSuggesterCommitCheckAsync.getSuggestionIndexSubspace(recordStore.indexSubspace(SIMPLE_TEXT_WITH_AUTO_COMPLETE), TupleHelpers.EMPTY), context, "_0.cfs", true);
}
}
use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.
the class FDBMetaDataStoreTest method updateRecordsWithNewUnionField.
@ParameterizedTest(name = "updateRecordsWithNewUnionField [reorderFields = {0}]")
@BooleanSource
public void updateRecordsWithNewUnionField(boolean reorderFields) {
try (FDBRecordContext context = fdb.openContext()) {
openMetaDataStore(context);
RecordMetaData metaData = RecordMetaData.build(TestRecords1Proto.getDescriptor());
metaDataStore.saveRecordMetaData(metaData);
context.commit();
assertNotNull(metaDataStore.getRecordMetaData().getRecordType("MySimpleRecord"));
}
try (FDBRecordContext context = fdb.openContext()) {
openMetaDataStore(context);
RecordMetaData oldMetaData = metaDataStore.getRecordMetaData();
metaDataStore.mutateMetaData(metaDataProtoBuilder -> {
final DescriptorProtos.FileDescriptorProto.Builder records = metaDataProtoBuilder.getRecordsBuilder();
records.getMessageTypeBuilderList().stream().filter(message -> message.getName().equals(RecordMetaDataBuilder.DEFAULT_UNION_NAME)).forEach(unionMessage -> {
unionMessage.getFieldBuilderList().stream().filter(field -> field.getName().equals("_MySimpleRecord")).forEach(field -> field.setName("_MySimpleRecord_old"));
int newFieldNumber = unionMessage.getFieldBuilderList().stream().mapToInt(DescriptorProtos.FieldDescriptorProto.Builder::getNumber).max().orElse(0) + 1;
DescriptorProtos.FieldDescriptorProto newField = DescriptorProtos.FieldDescriptorProto.newBuilder().setLabel(DescriptorProtos.FieldDescriptorProto.Label.LABEL_OPTIONAL).setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_MESSAGE).setTypeName("." + TestRecords1Proto.MySimpleRecord.getDescriptor().getFullName()).setName("_MySimpleRecord_new").setNumber(newFieldNumber).build();
if (reorderFields) {
List<DescriptorProtos.FieldDescriptorProto> fieldList = new ArrayList<>(unionMessage.getFieldBuilderList().size() + 1);
fieldList.add(newField);
fieldList.addAll(unionMessage.getFieldList());
unionMessage.clearField();
unionMessage.addAllField(fieldList);
} else {
unionMessage.addField(newField);
}
});
});
RecordMetaData newMetaData = metaDataStore.getRecordMetaData();
RecordType oldSimpleRecord = oldMetaData.getRecordType("MySimpleRecord");
assertEquals(TestRecords1EvolvedProto.RecordTypeUnion._MYSIMPLERECORD_FIELD_NUMBER, oldMetaData.getUnionFieldForRecordType(oldSimpleRecord).getNumber());
RecordType newSimpleRecord = newMetaData.getRecordType("MySimpleRecord");
assertSame(newMetaData.getUnionDescriptor().findFieldByName("_MySimpleRecord_new"), newMetaData.getUnionFieldForRecordType(newSimpleRecord));
assertThat(oldMetaData.getUnionFieldForRecordType(oldSimpleRecord).getNumber(), lessThan(newMetaData.getUnionFieldForRecordType(newSimpleRecord).getNumber()));
assertEquals(oldSimpleRecord.getSinceVersion(), newSimpleRecord.getSinceVersion());
context.commit();
}
try (FDBRecordContext context = fdb.openContext()) {
openMetaDataStore(context);
RecordMetaData metaData = metaDataStore.getRecordMetaData();
RecordType simpleRecord = metaData.getRecordType("MySimpleRecord");
assertEquals("_MySimpleRecord_new", metaData.getUnionFieldForRecordType(simpleRecord).getName());
int newFieldNumber = TestRecords1Proto.RecordTypeUnion.getDescriptor().getFields().stream().mapToInt(Descriptors.FieldDescriptor::getNumber).max().orElse(0) + 1;
assertEquals(newFieldNumber, metaData.getUnionFieldForRecordType(simpleRecord).getNumber());
}
}
use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.
the class FDBMetaDataStoreTest method assertDeprecated.
private static void assertDeprecated(@Nonnull RecordMetaData metaData, @Nonnull String recordType) {
RecordType recordTypeObj = metaData.getRecordType(recordType);
assertTrue(metaData.getUnionFieldForRecordType(recordTypeObj).getOptions().getDeprecated());
}
use of com.apple.foundationdb.record.metadata.RecordType in project fdb-record-layer by FoundationDB.
the class LuceneIndexTest method searchForAutoCompleteCrossingMultipleFields.
@Test
void searchForAutoCompleteCrossingMultipleFields() throws Exception {
try (FDBRecordContext context = openContext()) {
openRecordStore(context, metaDataBuilder -> {
metaDataBuilder.removeIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME);
metaDataBuilder.addIndex(COMPLEX_DOC, COMPLEX_MULTIPLE_TEXT_INDEXES_WITH_AUTO_COMPLETE);
});
// Write 8 texts and 6 of them contain the key "good"
recordStore.saveRecord(createComplexDocument(1623L, "Good morning", "", 1));
recordStore.saveRecord(createComplexDocument(1624L, "Good afternoon", "", 1));
recordStore.saveRecord(createComplexDocument(1625L, "good evening", "", 1));
recordStore.saveRecord(createComplexDocument(1626L, "Good night", "", 1));
recordStore.saveRecord(createComplexDocument(1627L, "", "That's really good!", 1));
recordStore.saveRecord(createComplexDocument(1628L, "", "I'm good", 1));
recordStore.saveRecord(createComplexDocument(1629L, "", "Hello Record Layer", 1));
RecordType recordType = recordStore.saveRecord(createComplexDocument(1630L, "", "Hello FoundationDB!", 1)).getRecordType();
List<IndexEntry> results = recordStore.scanIndex(COMPLEX_MULTIPLE_TEXT_INDEXES_WITH_AUTO_COMPLETE, IndexScanType.BY_LUCENE_AUTO_COMPLETE, TupleRange.allOf(Tuple.from("good")), null, ScanProperties.FORWARD_SCAN).asList().get();
results.stream().forEach(i -> assertDocumentPartialRecordFromIndexEntry(recordType, i, (String) i.getKey().get(i.getKeySize() - 1), (String) i.getKey().get(i.getKeySize() - 2), IndexScanType.BY_LUCENE_AUTO_COMPLETE));
assertEquals(6, context.getTimer().getCounter(FDBStoreTimer.Counts.LUCENE_SCAN_MATCHED_AUTO_COMPLETE_SUGGESTIONS).getCount());
assertEntriesAndSegmentInfoStoredInCompoundFile(AutoCompleteSuggesterCommitCheckAsync.getSuggestionIndexSubspace(recordStore.indexSubspace(COMPLEX_MULTIPLE_TEXT_INDEXES_WITH_AUTO_COMPLETE), TupleHelpers.EMPTY), context, "_0.cfs", true);
// Assert the count of suggestions
assertEquals(6, results.size());
// Assert the suggestions' keys
List<String> suggestions = results.stream().map(i -> (String) i.getKey().get(i.getKeySize() - 1)).collect(Collectors.toList());
assertEquals(ImmutableList.of("good evening", "Good night", "Good morning", "Good afternoon", "I'm good", "That's really good!"), suggestions);
// Assert the corresponding field for the suggestions
List<String> fields = results.stream().map(i -> (String) i.getKey().get(i.getKeySize() - 2)).collect(Collectors.toList());
assertEquals(ImmutableList.of("text", "text", "text", "text", "text2", "text2"), fields);
// Assert the suggestions are sorted according to their values, which are determined by the position of the term into the indexed text
List<Long> values = results.stream().map(i -> (Long) i.getValue().get(0)).collect(Collectors.toList());
List<Long> valuesSorted = new ArrayList<>(values);
Collections.sort(valuesSorted, Collections.reverseOrder());
assertEquals(valuesSorted, values);
assertEquals(values.get(0), values.get(1));
assertEquals(values.get(1), values.get(2));
assertEquals(values.get(2), values.get(3));
assertTrue(values.get(3) > values.get(4));
assertTrue(values.get(4) > values.get(5));
}
}
Aggregations