Examples with RecordCoreException com.apple.foundationdb.record.RecordCoreException used on opensource projects

Example 86 with RecordCoreException

use of com.apple.foundationdb.record.RecordCoreException in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreStateCacheTest method setCacheableAtWrongFormatVersion.

@Test
public void setCacheableAtWrongFormatVersion() throws Exception {
    FDBRecordStoreStateCache currentCache = fdb.getStoreStateCache();
    try {
        fdb.setStoreStateCache(metaDataVersionStampCacheFactory.getCache(fdb));
        // Initialize the store at the format version prior to the cacheable state version
        FDBRecordStore.Builder storeBuilder = FDBRecordStore.newBuilder().setKeySpacePath(path).setMetaDataProvider(RecordMetaData.build(TestRecords1Proto.getDescriptor())).setFormatVersion(FDBRecordStore.CACHEABLE_STATE_FORMAT_VERSION - 1);
        try (FDBRecordContext context = openContext()) {
            storeBuilder.copyBuilder().setContext(context).create();
            commit(context);
        }
        try (FDBRecordContext context = openContext()) {
            FDBRecordStore recordStore = storeBuilder.copyBuilder().setContext(context).open();
            assertEquals(FDBRecordStore.CACHEABLE_STATE_FORMAT_VERSION - 1, recordStore.getFormatVersion());
            RecordCoreException e = assertThrows(RecordCoreException.class, () -> recordStore.setStateCacheability(true));
            assertThat(e.getMessage(), containsString("cannot mark record store state cacheable at format version"));
            commit(context);
        }
        // Update the format version
        try (FDBRecordContext context = openContext()) {
            storeBuilder.copyBuilder().setContext(context).setFormatVersion(FDBRecordStore.CACHEABLE_STATE_FORMAT_VERSION).open();
            commit(context);
        }
        try (FDBRecordContext context = openContext()) {
            // Assert that format version happens because of the upgrade behind the scenes
            assertEquals(FDBRecordStore.CACHEABLE_STATE_FORMAT_VERSION - 1, storeBuilder.getFormatVersion());
            FDBRecordStore recordStore = storeBuilder.copyBuilder().setContext(context).open();
            assertEquals(FDBRecordStore.CACHEABLE_STATE_FORMAT_VERSION, recordStore.getFormatVersion());
            assertTrue(recordStore.setStateCacheability(true));
            commit(context);
        }
    } finally {
        fdb.setStoreStateCache(currentCache);
    }
}

Example 87 with RecordCoreException

use of com.apple.foundationdb.record.RecordCoreException in project fdb-record-layer by FoundationDB.

the class LuceneIndexExpressions method getFieldsRecursively.

@SuppressWarnings("squid:S3776")
public static <T extends RecordSource<T>> void getFieldsRecursively(@Nonnull KeyExpression expression, @Nonnull T source, @Nonnull DocumentDestination<T> destination, @Nullable String fieldNamePrefix, int keyIndex, int groupingCount, @Nonnull List<Integer> overriddenKeyRanges) {
    if (expression instanceof ThenKeyExpression) {
        int count = 0;
        for (KeyExpression child : ((ThenKeyExpression) expression).getChildren()) {
            getFieldsRecursively(child, source, destination, fieldNamePrefix, keyIndex + count, groupingCount, overriddenKeyRanges);
            count += child.getColumnSize();
        }
        return;
    }
    String fieldNameSuffix = null;
    boolean suffixOverride = false;
    if (expression instanceof LuceneFunctionKeyExpression.LuceneFieldName) {
        LuceneFunctionKeyExpression.LuceneFieldName fieldNameExpression = (LuceneFunctionKeyExpression.LuceneFieldName) expression;
        KeyExpression nameExpression = fieldNameExpression.getNameExpression();
        if (nameExpression instanceof LiteralKeyExpression) {
            fieldNameSuffix = (String) ((LiteralKeyExpression<?>) nameExpression).getValue();
        } else if (nameExpression instanceof FieldKeyExpression) {
            Iterator<Object> names = source.getValues((FieldKeyExpression) nameExpression).iterator();
            if (names.hasNext()) {
                fieldNameSuffix = (String) names.next();
                if (names.hasNext()) {
                    throw new RecordCoreException("Lucene field name override should evaluate to single value");
                }
            }
        } else {
            throw new RecordCoreException("Lucene field name override should be a literal or a field");
        }
        suffixOverride = true;
        expression = fieldNameExpression.getNamedExpression();
    }
    if (expression instanceof NestingKeyExpression) {
        NestingKeyExpression nestingExpression = (NestingKeyExpression) expression;
        FieldKeyExpression parentExpression = nestingExpression.getParent();
        KeyExpression child = nestingExpression.getChild();
        if (!suffixOverride) {
            fieldNameSuffix = parentExpression.getFieldName();
        } else {
            addOverriddenKeyRange(overriddenKeyRanges, fieldNamePrefix, fieldNameSuffix);
        }
        String fieldName = appendFieldName(fieldNamePrefix, fieldNameSuffix);
        for (T subsource : source.getChildren(parentExpression)) {
            getFieldsRecursively(child, subsource, destination, fieldName, keyIndex, groupingCount, overriddenKeyRanges);
        }
        if (suffixOverride) {
            // Remove the last 2 numbers added above
            removedLastOverriddenKeyRange(overriddenKeyRanges);
        }
        return;
    }
    boolean fieldStored = false;
    boolean fieldText = false;
    while (true) {
        if (expression instanceof LuceneFunctionKeyExpression.LuceneStored) {
            LuceneFunctionKeyExpression.LuceneStored storedExpression = (LuceneFunctionKeyExpression.LuceneStored) expression;
            fieldStored = true;
            expression = storedExpression.getStoredExpression();
        } else if (expression instanceof LuceneFunctionKeyExpression.LuceneText) {
            LuceneFunctionKeyExpression.LuceneText textExpression = (LuceneFunctionKeyExpression.LuceneText) expression;
            fieldText = true;
            expression = textExpression.getFieldExpression();
        } else {
            // TODO: More text options.
            break;
        }
    }
    if (expression instanceof FieldKeyExpression) {
        FieldKeyExpression fieldExpression = (FieldKeyExpression) expression;
        if (!suffixOverride) {
            fieldNameSuffix = fieldExpression.getFieldName();
        } else {
            addOverriddenKeyRange(overriddenKeyRanges, fieldNamePrefix, fieldNameSuffix);
        }
        String fieldName = appendFieldName(fieldNamePrefix, fieldNameSuffix);
        if (fieldName == null) {
            fieldName = "_";
        }
        Descriptors.Descriptor recordDescriptor = source.getDescriptor();
        Descriptors.FieldDescriptor fieldDescriptor = recordDescriptor.findFieldByName(fieldExpression.getFieldName());
        DocumentFieldType fieldType;
        if (fieldText) {
            switch(fieldDescriptor.getJavaType()) {
                case STRING:
                    fieldType = DocumentFieldType.TEXT;
                    break;
                default:
                    throw new RecordCoreException("Unknown Lucene text field type");
            }
        } else {
            switch(fieldDescriptor.getJavaType()) {
                case STRING:
                    fieldType = DocumentFieldType.STRING;
                    break;
                case INT:
                    fieldType = DocumentFieldType.INT;
                    break;
                case LONG:
                    fieldType = DocumentFieldType.LONG;
                    break;
                case DOUBLE:
                    fieldType = DocumentFieldType.DOUBLE;
                    break;
                case BOOLEAN:
                    fieldType = DocumentFieldType.BOOLEAN;
                    break;
                default:
                    throw new RecordCoreException("Unknown Lucene field type");
            }
        }
        for (Object value : source.getValues(fieldExpression)) {
            destination.addField(source, fieldName, value, fieldType, fieldStored, overriddenKeyRanges, keyIndex < groupingCount ? keyIndex : -1);
        }
        if (suffixOverride) {
            // Remove the last 2 numbers added above
            removedLastOverriddenKeyRange(overriddenKeyRanges);
        }
        return;
    }
    throw new RecordCoreException("Unknown Lucene field key expression");
}

Example 88 with RecordCoreException

use of com.apple.foundationdb.record.RecordCoreException in project fdb-record-layer by FoundationDB.

the class LuceneIndexMaintainer method addTermToSuggesterIfNeeded.

private boolean addTermToSuggesterIfNeeded(@Nonnull String value, @Nonnull String fieldName, @Nullable AnalyzingInfixSuggester suggester, @Nullable Tuple groupingKey) {
    if (suggester == null) {
        return false;
    }
    final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
    final RecordLayerPropertyKey<Integer> sizeLimitProp = LuceneRecordContextProperties.LUCENE_AUTO_COMPLETE_TEXT_SIZE_UPPER_LIMIT;
    final int sizeLimit = Objects.requireNonNullElse(state.context.getPropertyStorage().getPropertyValue(sizeLimitProp), sizeLimitProp.getDefaultValue()).intValue();
    // Ignore this text if its size exceeds the limitation
    if (valueBytes.length > sizeLimit) {
        if (LOG.isTraceEnabled()) {
            LOG.trace(KeyValueLogMessage.of("Skip auto-complete indexing due to exceeding size limitation", LogMessageKeys.DATA_SIZE, valueBytes.length, LogMessageKeys.DATA_VALUE, value.substring(0, Math.min(value.length(), 100)), LogMessageKeys.FIELD_NAME, fieldName, LogMessageKeys.GROUPING_KEY, groupingKey));
        }
        return false;
    }
    try {
        suggester.add(new BytesRef(valueBytes), Set.of(new BytesRef(fieldName.getBytes(StandardCharsets.UTF_8))), state.context.getPropertyStorage().getPropertyValue(LuceneRecordContextProperties.LUCENE_AUTO_COMPLETE_DEFAULT_WEIGHT), new BytesRef(groupingKey == null ? Tuple.from(fieldName).pack() : groupingKey.add(fieldName).pack()));
        if (LOG.isTraceEnabled()) {
            LOG.trace(KeyValueLogMessage.of("Added auto-complete suggestion to suggester", LogMessageKeys.DATA_SIZE, valueBytes.length, LogMessageKeys.DATA_VALUE, value.substring(0, Math.min(value.length(), 100)), LogMessageKeys.FIELD_NAME, fieldName, LogMessageKeys.GROUPING_KEY, groupingKey));
        }
        return true;
    } catch (IOException ex) {
        throw new RecordCoreException("Exception to add term into suggester", ex).addLogInfo(LogMessageKeys.INDEX_NAME, state.index.getName());
    }
}

Example 89 with RecordCoreException

use of com.apple.foundationdb.record.RecordCoreException in project fdb-record-layer by FoundationDB.

the class LuceneIndexQueryPlan method executePlan.

/**
 * Override here to have specific logic to build the {@link QueryResult} for lucene auto complete suggestion result.
 */
@Nonnull
@Override
public <M extends Message> RecordCursor<QueryResult> executePlan(@Nonnull FDBRecordStoreBase<M> store, @Nonnull EvaluationContext context, @Nullable byte[] continuation, @Nonnull ExecuteProperties executeProperties) {
    final RecordMetaData metaData = store.getRecordMetaData();
    final Index index = metaData.getIndex(indexName);
    final Collection<RecordType> recordTypes = metaData.recordTypesForIndex(index);
    if (recordTypes.size() != 1) {
        throw new RecordCoreException("No lucene index should span multiple record types");
    }
    final IndexScanType scanType = getScanType();
    if (scanType == IndexScanType.BY_LUCENE_AUTO_COMPLETE || scanType == IndexScanType.BY_LUCENE_SPELLCHECK) {
        final RecordType recordType = recordTypes.iterator().next();
        final RecordCursor<IndexEntry> entryRecordCursor = executeEntries(store, context, continuation, executeProperties);
        return entryRecordCursor.map(QueryPlanUtils.getCoveringIndexEntryToPartialRecordFunction(store, recordType.getName(), indexName, getToPartialRecord(index, recordType, scanType), scanType)).map(QueryResult::of);
    }
    return super.executePlan(store, context, continuation, executeProperties);
}

Example 90 with RecordCoreException

use of com.apple.foundationdb.record.RecordCoreException in project fdb-record-layer by FoundationDB.

the class LuceneRecordCursor method onNext.

@Nonnull
@Override
public CompletableFuture<RecordCursorResult<IndexEntry>> onNext() {
    if (nextResult != null && !nextResult.hasNext()) {
        // hasNext is false to avoid the NoNextReason changing.
        return CompletableFuture.completedFuture(nextResult);
    }
    if (topDocs == null) {
        try {
            performScan();
        } catch (IOException ioException) {
            throw new RuntimeException(ioException);
        }
    }
    if (topDocs.scoreDocs.length - 1 < currentPosition && limitRemaining > 0 && !exhausted) {
        try {
            performScan();
        } catch (IOException ioException) {
            throw new RuntimeException(ioException);
        }
        currentPosition = Math.max(currentPosition - MAX_PAGE_SIZE, 0);
    }
    if (limitRemaining > 0 && currentPosition < topDocs.scoreDocs.length && limitManager.tryRecordScan()) {
        return CompletableFuture.supplyAsync(() -> {
            try {
                Document document = searcher.doc(topDocs.scoreDocs[currentPosition].doc);
                searchAfter = topDocs.scoreDocs[currentPosition];
                IndexableField primaryKey = document.getField(LuceneIndexMaintainer.PRIMARY_KEY_FIELD_NAME);
                BytesRef pk = primaryKey.binaryValue();
                if (LOGGER.isTraceEnabled()) {
                    LOGGER.trace("document={}", document);
                    LOGGER.trace("primary key read={}", Tuple.fromBytes(pk.bytes, pk.offset, pk.length));
                }
                if (timer != null) {
                    timer.increment(FDBStoreTimer.Counts.LOAD_SCAN_ENTRY);
                }
                if (limitRemaining != Integer.MAX_VALUE) {
                    limitRemaining--;
                }
                List<Object> setPrimaryKey = Tuple.fromBytes(pk.bytes).getItems();
                List<Object> fieldValues = Lists.newArrayList(fields);
                int[] keyPos = state.index.getPrimaryKeyComponentPositions();
                Tuple tuple;
                if (keyPos != null) {
                    List<Object> leftovers = Lists.newArrayList();
                    for (int i = 0; i < keyPos.length; i++) {
                        if (keyPos[i] > -1) {
                            fieldValues.set(keyPos[i], setPrimaryKey.get(i));
                        } else {
                            leftovers.add(setPrimaryKey.get(i));
                        }
                    }
                    tuple = Tuple.fromList(fieldValues).addAll(leftovers);
                } else {
                    tuple = Tuple.fromList(fieldValues).addAll(setPrimaryKey);
                }
                nextResult = RecordCursorResult.withNextValue(new IndexEntry(state.index, tuple, null), continuationHelper());
                currentPosition++;
                return nextResult;
            } catch (Exception e) {
                throw new RecordCoreException("Failed to get document", "currentPosition", currentPosition, "exception", e);
            }
        }, executor);
    } else {
        // a limit was exceeded
        if (limitRemaining <= 0) {
            nextResult = RecordCursorResult.withoutNextValue(continuationHelper(), NoNextReason.RETURN_LIMIT_REACHED);
        } else if (currentPosition >= topDocs.scoreDocs.length) {
            nextResult = RecordCursorResult.withoutNextValue(continuationHelper(), NoNextReason.SOURCE_EXHAUSTED);
        } else {
            final Optional<NoNextReason> stoppedReason = limitManager.getStoppedReason();
            if (!stoppedReason.isPresent()) {
                throw new RecordCoreException("limit manager stopped LuceneRecordCursor but did not report a reason");
            } else {
                nextResult = RecordCursorResult.withoutNextValue(continuationHelper(), stoppedReason.get());
            }
        }
        return CompletableFuture.completedFuture(nextResult);
    }
}