Examples with Key com.apple.foundationdb.record.metadata.Key used on opensource projects

Example 6 with Key

use of com.apple.foundationdb.record.metadata.Key in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method loadRecordVersionAsync.

/**
 * Async version of {@link #loadRecordVersion(Tuple, boolean)}.
 * @param primaryKey the primary key of the record
 * @param snapshot whether to snapshot read
 * @return a future that completes with the version of the record of {@code Optional.empty()} if versions are not enabled for this store
 */
@Nonnull
public Optional<CompletableFuture<FDBRecordVersion>> loadRecordVersionAsync(@Nonnull final Tuple primaryKey, final boolean snapshot) {
    final RecordMetaData metaData = metaDataProvider.getRecordMetaData();
    if (useOldVersionFormat() && !metaData.isStoreRecordVersions()) {
        // a priori that this will return an empty optional, so we return it without doing any I/O.
        return Optional.empty();
    } else {
        byte[] versionKey = getSubspace().pack(recordVersionKey(primaryKey));
        Optional<CompletableFuture<FDBRecordVersion>> cachedOptional = context.getLocalVersion(versionKey).map(localVersion -> CompletableFuture.completedFuture(FDBRecordVersion.incomplete(localVersion)));
        if (cachedOptional.isPresent()) {
            return cachedOptional;
        }
        final ReadTransaction tr = snapshot ? ensureContextActive().snapshot() : ensureContextActive();
        return Optional.of(tr.get(versionKey).thenApply(valueBytes -> {
            if (valueBytes == null) {
                return null;
            } else if (useOldVersionFormat()) {
                return FDBRecordVersion.complete(valueBytes, false);
            } else {
                return SplitHelper.unpackVersion(valueBytes);
            }
        }));
    }
}

Example 7 with Key

use of com.apple.foundationdb.record.metadata.Key in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method checkRebuildIndexes.

private CompletableFuture<Void> checkRebuildIndexes(@Nullable UserVersionChecker userVersionChecker, @Nonnull RecordMetaDataProto.DataStoreInfo.Builder info, int oldFormatVersion, @Nonnull RecordMetaData metaData, int oldMetaDataVersion, boolean rebuildRecordCounts, List<CompletableFuture<Void>> work) {
    final boolean newStore = oldFormatVersion == 0;
    final Map<Index, List<RecordType>> indexes = metaData.getIndexesToBuildSince(oldMetaDataVersion);
    if (!indexes.isEmpty()) {
        // If all the new indexes are only for a record type whose primary key has a type prefix, then we can scan less.
        RecordType singleRecordTypeWithPrefixKey = singleRecordTypeWithPrefixKey(indexes);
        final AtomicLong recordCountRef = new AtomicLong(-1);
        final Supplier<CompletableFuture<Long>> lazyRecordCount = getAndRememberFutureLong(recordCountRef, () -> getRecordCountForRebuildIndexes(newStore, rebuildRecordCounts, indexes, singleRecordTypeWithPrefixKey));
        AtomicLong recordsSizeRef = new AtomicLong(-1);
        final Supplier<CompletableFuture<Long>> lazyRecordsSize = getAndRememberFutureLong(recordsSizeRef, () -> getRecordSizeForRebuildIndexes(singleRecordTypeWithPrefixKey));
        if (singleRecordTypeWithPrefixKey == null && formatVersion >= SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION && omitUnsplitRecordSuffix) {
            // Check to see if the unsplit format can be upgraded on an empty store.
            // Only works if singleRecordTypeWithPrefixKey is null as otherwise, the recordCount will not contain
            // all records
            work.add(lazyRecordCount.get().thenAccept(recordCount -> {
                if (recordCount == 0) {
                    if (newStore ? LOGGER.isDebugEnabled() : LOGGER.isInfoEnabled()) {
                        KeyValueLogMessage msg = KeyValueLogMessage.build("upgrading unsplit format on empty store", LogMessageKeys.NEW_FORMAT_VERSION, formatVersion, subspaceProvider.logKey(), subspaceProvider.toString(context));
                        if (newStore) {
                            LOGGER.debug(msg.toString());
                        } else {
                            LOGGER.info(msg.toString());
                        }
                    }
                    omitUnsplitRecordSuffix = formatVersion < SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION;
                    info.clearOmitUnsplitRecordSuffix();
                    // We used snapshot to determine emptiness, and are now acting on it.
                    addRecordsReadConflict();
                }
            }));
        }
        Map<Index, CompletableFuture<IndexState>> newStates = getStatesForRebuildIndexes(userVersionChecker, indexes, lazyRecordCount, lazyRecordsSize, newStore, oldMetaDataVersion, oldFormatVersion);
        return rebuildIndexes(indexes, newStates, work, newStore ? RebuildIndexReason.NEW_STORE : RebuildIndexReason.FEW_RECORDS, oldMetaDataVersion).thenRun(() -> {
            // Log after checking all index states
            maybeLogIndexesNeedingRebuilding(newStates, recordCountRef, recordsSizeRef, rebuildRecordCounts, newStore);
            context.increment(FDBStoreTimer.Counts.INDEXES_NEED_REBUILDING, newStates.entrySet().size());
        });
    } else {
        return work.isEmpty() ? AsyncUtil.DONE : AsyncUtil.whenAll(work);
    }
}

Example 8 with Key

use of com.apple.foundationdb.record.metadata.Key in project fdb-record-layer by FoundationDB.

the class IndexFunctionHelper method recordFunctionIndexEntry.

/**
 * Get the index entry for use by the given index to evaluate the given record function.
 *
 * In most cases, this is the same as {@link KeyExpression#evaluateSingleton}. But if {@code recordFunction} is
 * the result of {@link #recordFunctionWithSubrecordCondition}, a matching entry will be found.
 * @param store store against which function will be evaluated
 * @param index index for which to evaluate
 * @param context context for parameter bindings
 * @param recordFunction record function for which to evaluate
 * @param record record against which to evaluate
 * @param groupSize grouping size for the given index
 * @return an index entry or {@code null} if none matches a bound condition
 */
@Nullable
public static Key.Evaluated recordFunctionIndexEntry(@Nonnull FDBRecordStore store, @Nonnull Index index, @Nonnull EvaluationContext context, @Nullable IndexRecordFunction<?> recordFunction, @Nonnull FDBRecord<?> record, int groupSize) {
    final KeyExpression expression = index.getRootExpression();
    if (!(recordFunction instanceof IndexRecordFunctionWithSubrecordValues)) {
        return expression.evaluateSingleton(record);
    }
    final IndexRecordFunctionWithSubrecordValues<?> recordFunctionWithSubrecordValues = (IndexRecordFunctionWithSubrecordValues<?>) recordFunction;
    final int scalarPrefixCount = recordFunctionWithSubrecordValues.getScalarPrefixCount();
    final List<Object> toMatch = recordFunctionWithSubrecordValues.getValues(store, context).values();
    List<Object> prev = null;
    Key.Evaluated match = null;
    for (Key.Evaluated key : expression.evaluate(record)) {
        final List<Object> subrecord = key.values();
        for (int i = 0; i < groupSize; i++) {
            if (i < scalarPrefixCount) {
                if (prev != null) {
                    if (!Objects.equals(prev.get(i), subrecord.get(i))) {
                        throw new RecordCoreException("All subrecords should match for non-constrained keys");
                    }
                }
            } else {
                if (toMatch.get(i - scalarPrefixCount).equals(subrecord.get(i))) {
                    if (match != null) {
                        throw new RecordCoreException("More than one matching subrecord");
                    }
                    match = key;
                }
            }
        }
        prev = subrecord;
    }
    return match;
}

Example 9 with Key

use of com.apple.foundationdb.record.metadata.Key in project fdb-record-layer by FoundationDB.

the class ImplementInUnionRule method onMatch.

@SuppressWarnings({ "unchecked", "java:S135" })
@Override
public void onMatch(@Nonnull PlannerRuleCall call) {
    final var context = call.getContext();
    final var bindings = call.getBindings();
    final var requestedOrderingsOptional = call.getInterestingProperty(OrderingAttribute.ORDERING);
    if (requestedOrderingsOptional.isEmpty()) {
        return;
    }
    final var requestedOrderings = requestedOrderingsOptional.get();
    final var commonPrimaryKey = context.getCommonPrimaryKey();
    if (commonPrimaryKey == null) {
        return;
    }
    final var selectExpression = bindings.get(root);
    if (!selectExpression.getPredicates().isEmpty()) {
        return;
    }
    final var explodeQuantifiers = bindings.get(explodeQuantifiersMatcher);
    if (explodeQuantifiers.isEmpty()) {
        return;
    }
    final var explodeAliases = Quantifiers.aliases(explodeQuantifiers);
    final var innerQuantifierOptional = findInnerQuantifier(selectExpression, explodeQuantifiers, explodeAliases);
    if (innerQuantifierOptional.isEmpty()) {
        return;
    }
    final var innerQuantifier = innerQuantifierOptional.get();
    final List<? extends Value> resultValues = selectExpression.getResultValues();
    if (resultValues.stream().anyMatch(resultValue -> !(resultValue instanceof QuantifiedColumnValue) || !((QuantifiedColumnValue) resultValue).getAlias().equals(innerQuantifier.getAlias()))) {
        return;
    }
    final var explodeExpressions = bindings.getAll(explodeExpressionMatcher);
    final var quantifierToExplodeBiMap = computeQuantifierToExplodeMap(explodeQuantifiers, explodeExpressions.stream().collect(LinkedIdentitySet.toLinkedIdentitySet()));
    final var explodeToQuantifierBiMap = quantifierToExplodeBiMap.inverse();
    final var sourcesBuilder = ImmutableList.<InSource>builder();
    for (final var explodeExpression : explodeExpressions) {
        final var explodeQuantifier = Objects.requireNonNull(explodeToQuantifierBiMap.getUnwrapped(explodeExpression));
        final List<? extends Value> explodeResultValues = explodeExpression.getResultValues();
        if (explodeResultValues.size() != 1) {
            return;
        }
        final Value explodeValue = Iterables.getOnlyElement(explodeResultValues);
        // 
        // Create the source for the in-union plan
        // 
        final InSource inSource;
        if (explodeValue instanceof LiteralValue<?>) {
            final Object literalValue = ((LiteralValue<?>) explodeValue).getLiteralValue();
            if (literalValue instanceof List<?>) {
                inSource = new InValuesSource(CORRELATION.bindingName(explodeQuantifier.getAlias().getId()), (List<Object>) literalValue);
            } else {
                return;
            }
        } else if (explodeValue instanceof QuantifiedColumnValue) {
            inSource = new InParameterSource(CORRELATION.bindingName(explodeQuantifier.getAlias().getId()), ((QuantifiedColumnValue) explodeValue).getAlias().getId());
        } else {
            return;
        }
        sourcesBuilder.add(inSource);
    }
    final var inSources = sourcesBuilder.build();
    final Map<Ordering, ImmutableList<RecordQueryPlan>> groupedByOrdering = innerQuantifier.getRangesOver().getMembers().stream().flatMap(relationalExpression -> relationalExpression.narrowMaybe(RecordQueryPlan.class).stream()).flatMap(plan -> {
        final Optional<Ordering> orderingForLegOptional = OrderingProperty.evaluate(plan, context);
        return orderingForLegOptional.stream().map(ordering -> Pair.of(ordering, plan));
    }).collect(Collectors.groupingBy(Pair::getLeft, Collectors.mapping(Pair::getRight, ImmutableList.toImmutableList())));
    final int attemptFailedInJoinAsUnionMaxSize = call.getContext().getPlannerConfiguration().getAttemptFailedInJoinAsUnionMaxSize();
    for (final Map.Entry<Ordering, ImmutableList<RecordQueryPlan>> providedOrderingEntry : groupedByOrdering.entrySet()) {
        for (final RequestedOrdering requestedOrdering : requestedOrderings) {
            final var providedOrdering = providedOrderingEntry.getKey();
            final var matchingKeyExpressionsBuilder = ImmutableSet.<KeyExpression>builder();
            for (Map.Entry<KeyExpression, Comparisons.Comparison> expressionComparisonEntry : providedOrdering.getEqualityBoundKeyMap().entries()) {
                final Comparisons.Comparison comparison = expressionComparisonEntry.getValue();
                if (comparison.getType() == Comparisons.Type.EQUALS && comparison instanceof Comparisons.ParameterComparison) {
                    final Comparisons.ParameterComparison parameterComparison = (Comparisons.ParameterComparison) comparison;
                    if (parameterComparison.isCorrelation() && explodeAliases.containsAll(parameterComparison.getCorrelatedTo())) {
                        matchingKeyExpressionsBuilder.add(expressionComparisonEntry.getKey());
                    }
                }
            }
            // Compute a comparison key that satisfies the requested ordering
            final Optional<Ordering> combinedOrderingOptional = orderingForInUnion(providedOrdering, requestedOrdering, matchingKeyExpressionsBuilder.build());
            if (combinedOrderingOptional.isEmpty()) {
                continue;
            }
            final Ordering combinedOrdering = combinedOrderingOptional.get();
            final List<KeyPart> orderingKeyParts = combinedOrdering.getOrderingKeyParts();
            final List<KeyExpression> orderingKeys = orderingKeyParts.stream().map(KeyPart::getNormalizedKeyExpression).collect(ImmutableList.toImmutableList());
            // 
            // At this point we know we can implement the distinct union over the partitions of compatibly ordered plans
            // 
            final KeyExpression comparisonKey = orderingKeys.size() == 1 ? Iterables.getOnlyElement(orderingKeys) : Key.Expressions.concat(orderingKeys);
            final GroupExpressionRef<RecordQueryPlan> newInnerPlanReference = GroupExpressionRef.from(providedOrderingEntry.getValue());
            final Quantifier.Physical newInnerQuantifier = Quantifier.physical(newInnerPlanReference);
            call.yield(call.ref(new RecordQueryInUnionPlan(newInnerQuantifier, inSources, comparisonKey, RecordQueryUnionPlanBase.isReversed(ImmutableList.of(newInnerQuantifier)), attemptFailedInJoinAsUnionMaxSize)));
        }
    }
}

Example 10 with Key

use of com.apple.foundationdb.record.metadata.Key in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method scanIndexWithValue.

/**
 * Verify that explicit (i.e. bypassing the planner) index scans work .
 */
@Test
public void scanIndexWithValue() throws Exception {
    RecordMetaDataHook hook = metaData -> {
        metaData.removeIndex("MySimpleRecord$num_value_unique");
        metaData.addIndex("MySimpleRecord", new Index("multi_index_value", Key.Expressions.field("num_value_unique"), Key.Expressions.field("num_value_2"), IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
    };
    complexQuerySetup(hook);
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<IndexEntry> cursor = recordStore.scanIndex(recordStore.getRecordMetaData().getIndex("multi_index_value"), IndexScanType.BY_VALUE, new TupleRange(Tuple.from(900L), Tuple.from(950L), EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_INCLUSIVE), null, ScanProperties.FORWARD_SCAN).asIterator()) {
            while (cursor.hasNext()) {
                IndexEntry tuples = cursor.next();
                Tuple key = tuples.getKey();
                Tuple value = tuples.getValue();
                assertEquals(2, key.size());
                assertEquals(1, value.size());
                assertTrue(key.getLong(0) >= 900);
                assertTrue(key.getLong(0) <= 950);
                assertTrue(value.getLong(0) == (999 - i) % 3);
                i++;
            }
        }
        assertEquals(50, i);
        assertDiscardedNone(context);
    }
}