Examples with Tuple com.apple.foundationdb.tuple.Tuple used on opensource projects

Example 11 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class VersionIndexMaintainer method updateOneKeyAsync.

// Called by updateIndexKeys in StandardIndexMaintainer.
@Override
protected <M extends Message> CompletableFuture<Void> updateOneKeyAsync(@Nonnull final FDBIndexableRecord<M> savedRecord, final boolean remove, @Nonnull final IndexEntry indexEntry) {
    if (state.index.isUnique()) {
        throw new MetaDataException(String.format("%s index %s does not support unique indexes", state.index.getType(), state.index.getName()));
    }
    final Tuple valueKey = indexEntry.getKey();
    final Tuple value = indexEntry.getValue();
    final long startTime = System.nanoTime();
    final Tuple entryKey = indexEntryKey(valueKey, savedRecord.getPrimaryKey());
    final boolean hasIncomplete = entryKey.hasIncompleteVersionstamp();
    final byte[] keyBytes;
    if (hasIncomplete) {
        keyBytes = state.indexSubspace.packWithVersionstamp(entryKey);
    } else {
        keyBytes = state.indexSubspace.pack(entryKey);
    }
    if (remove) {
        if (hasIncomplete) {
            state.context.removeVersionMutation(keyBytes);
        } else {
            state.transaction.clear(state.indexSubspace.pack(entryKey));
        }
        if (state.store.getTimer() != null) {
            state.store.getTimer().recordSinceNanoTime(FDBStoreTimer.Events.DELETE_INDEX_ENTRY, startTime);
        }
    } else {
        final byte[] valueBytes = value.pack();
        checkKeyValueSizes(savedRecord, valueKey, value, keyBytes, valueBytes);
        if (hasIncomplete) {
            state.context.addVersionMutation(MutationType.SET_VERSIONSTAMPED_KEY, keyBytes, valueBytes);
        } else {
            state.transaction.set(keyBytes, valueBytes);
        }
        if (state.store.getTimer() != null) {
            state.store.getTimer().recordSinceNanoTime(FDBStoreTimer.Events.SAVE_INDEX_ENTRY, startTime);
        }
    }
    return AsyncUtil.DONE;
}

Example 12 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class RankIndexMaintainer method updateIndexKeys.

@Override
protected <M extends Message> CompletableFuture<Void> updateIndexKeys(@Nonnull final FDBIndexableRecord<M> savedRecord, final boolean remove, @Nonnull final List<IndexEntry> indexEntries) {
    final int groupPrefixSize = getGroupingCount();
    final Subspace extraSubspace = getSecondarySubspace();
    final List<CompletableFuture<Void>> ordinaryIndexFutures = new ArrayList<>(indexEntries.size());
    final Map<Subspace, CompletableFuture<Void>> rankFutures = Maps.newHashMapWithExpectedSize(indexEntries.size());
    for (IndexEntry indexEntry : indexEntries) {
        // Maintain an ordinary B-tree index by score.
        CompletableFuture<Void> updateOrdinaryIndex = updateOneKeyAsync(savedRecord, remove, indexEntry);
        if (!MoreAsyncUtil.isCompletedNormally(updateOrdinaryIndex)) {
            ordinaryIndexFutures.add(updateOrdinaryIndex);
        }
        final Subspace rankSubspace;
        final Tuple scoreKey;
        if (groupPrefixSize > 0) {
            final List<Object> keyValues = indexEntry.getKey().getItems();
            rankSubspace = extraSubspace.subspace(Tuple.fromList(keyValues.subList(0, groupPrefixSize)));
            scoreKey = Tuple.fromList(keyValues.subList(groupPrefixSize, keyValues.size()));
        } else {
            rankSubspace = extraSubspace;
            scoreKey = indexEntry.getKey();
        }
        // It is unsafe to have two concurrent updates to the same ranked set, so ensure that at most
        // one update per grouping key is ongoing at any given time
        final Function<Void, CompletableFuture<Void>> futureSupplier = vignore -> RankedSetIndexHelper.updateRankedSet(state, rankSubspace, config, indexEntry.getKey(), scoreKey, remove);
        CompletableFuture<Void> existingFuture = rankFutures.get(rankSubspace);
        if (existingFuture == null) {
            rankFutures.put(rankSubspace, futureSupplier.apply(null));
        } else {
            rankFutures.put(rankSubspace, existingFuture.thenCompose(futureSupplier));
        }
    }
    return CompletableFuture.allOf(AsyncUtil.whenAll(ordinaryIndexFutures), AsyncUtil.whenAll(rankFutures.values()));
}

Example 13 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexMaintainer method scan.

@Nonnull
@Override
public RecordCursor<IndexEntry> scan(@Nonnull IndexScanType scanType, @Nonnull TupleRange range, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (scanType != IndexScanType.BY_GROUP) {
        throw new RecordCoreException("Can only scan bitmap index by group.");
    }
    final int groupPrefixSize = getGroupingCount();
    final long startPosition;
    if (range.getLow() != null && range.getLow().size() > groupPrefixSize && range.getLow().get(groupPrefixSize) != null) {
        if (range.getLowEndpoint() == EndpointType.RANGE_EXCLUSIVE) {
            startPosition = range.getLow().getLong(groupPrefixSize) + 1;
        } else {
            startPosition = range.getLow().getLong(groupPrefixSize);
        }
        if (startPosition % entrySize != 0) {
            range = new TupleRange(range.getLow().popBack().add(startPosition - Math.floorMod(startPosition, (long) entrySize)), range.getHigh(), EndpointType.RANGE_INCLUSIVE, range.getHighEndpoint());
        }
    } else {
        startPosition = Long.MIN_VALUE;
    }
    final long endPosition;
    if (range.getHigh() != null && range.getHigh().size() > groupPrefixSize && range.getHigh().get(groupPrefixSize) != null) {
        if (range.getHighEndpoint() == EndpointType.RANGE_INCLUSIVE) {
            endPosition = range.getHigh().getLong(groupPrefixSize) + 1;
        } else {
            endPosition = range.getHigh().getLong(groupPrefixSize);
        }
        if (endPosition % entrySize != 0) {
            range = new TupleRange(range.getLow(), range.getHigh().popBack().add(endPosition + Math.floorMod(entrySize - endPosition, (long) entrySize)), range.getLowEndpoint(), EndpointType.RANGE_INCLUSIVE);
        }
    } else {
        endPosition = Long.MAX_VALUE;
    }
    return scan(range, continuation, scanProperties).map(indexEntry -> {
        final long entryStart = indexEntry.getKey().getLong(groupPrefixSize);
        final byte[] entryBitmap = indexEntry.getValue().getBytes(0);
        final long entryEnd = entryStart + entryBitmap.length * 8;
        if (entryStart < startPosition || entryEnd > endPosition) {
            final long trimmedStart = Math.max(entryStart, startPosition);
            final long trimmedEnd = Math.min(entryEnd, endPosition);
            if (trimmedStart < trimmedEnd) {
                final Tuple trimmedKey = indexEntry.getKey().popBack().add(trimmedStart);
                final byte[] trimmedBitmap = new byte[((int) (trimmedEnd - trimmedStart) + 7) / 8];
                for (long i = trimmedStart; i < trimmedEnd; i++) {
                    int offset = (int) (i - entryStart);
                    if ((entryBitmap[offset / 8] & (byte) (1 << (offset % 8))) != 0) {
                        int trimmedOffset = (int) (i - trimmedStart);
                        trimmedBitmap[trimmedOffset / 8] |= (byte) (1 << (trimmedOffset % 8));
                    }
                }
                final Tuple subValue = Tuple.from(trimmedBitmap);
                return Optional.of(new IndexEntry(indexEntry.getIndex(), trimmedKey, subValue));
            } else {
                return Optional.<IndexEntry>empty();
            }
        } else {
            return Optional.of(indexEntry);
        }
    }).filter(Optional::isPresent).map(Optional::get);
}

Example 14 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexMaintainer method evaluateAggregateFunction.

@Override
@Nonnull
public CompletableFuture<Tuple> evaluateAggregateFunction(@Nonnull IndexAggregateFunction function, @Nonnull TupleRange range, @Nonnull IsolationLevel isolationveLevel) {
    if (!function.getName().equals(AGGREGATE_FUNCTION_NAME)) {
        throw new MetaDataException("this index does not support aggregate function: " + function);
    }
    final RecordCursor<IndexEntry> cursor = scan(IndexScanType.BY_GROUP, range, null, new ScanProperties(ExecuteProperties.newBuilder().setIsolationLevel(isolationveLevel).build()));
    final int groupPrefixSize = getGroupingCount();
    long startPosition = 0;
    if (range.getLow() != null && range.getLow().size() > groupPrefixSize) {
        startPosition = range.getLow().getLong(groupPrefixSize);
    }
    int size = entrySize;
    if (range.getHigh() != null && range.getHigh().size() > groupPrefixSize) {
        long endPosition = range.getHigh().getLong(groupPrefixSize);
        if (size > endPosition - startPosition) {
            // Narrow size to what can actually be passed through from scan.
            size = (int) (endPosition - startPosition);
        }
    }
    return cursor.reduce(new BitmapAggregator(startPosition, size), (combined, kv) -> combined.append(kv.getKey().getLong(kv.getKeySize() - 1), kv.getValue().getBytes(0))).thenApply(combined -> Tuple.from(combined.asByteArray()));
}

Example 15 with Tuple

use of com.apple.foundationdb.tuple.Tuple in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexMaintainer method updateIndexKeys.

@Override
@Nonnull
protected <M extends Message> CompletableFuture<Void> updateIndexKeys(@Nonnull final FDBIndexableRecord<M> savedRecord, final boolean remove, @Nonnull final List<IndexEntry> indexEntries) {
    final int groupPrefixSize = getGroupingCount();
    final List<CompletableFuture<Void>> futures = unique && !remove ? new ArrayList<>(indexEntries.size()) : null;
    for (IndexEntry indexEntry : indexEntries) {
        final long startTime = System.nanoTime();
        final Tuple groupKey = TupleHelpers.subTuple(indexEntry.getKey(), 0, groupPrefixSize);
        Object positionObject = indexEntry.getKey().get(groupPrefixSize);
        if (positionObject == null) {
            continue;
        }
        if (!(positionObject instanceof Number)) {
            // This should be prevented by the checks in the meta-data builder, but just in case
            throw new RecordCoreException("position field in index entry is not a number").addLogInfo(LogMessageKeys.KEY, indexEntry.getKey(), LogMessageKeys.INDEX_NAME, state.index.getName(), LogMessageKeys.INDEX_TYPE, state.index.getType());
        }
        long position = ((Number) positionObject).longValue();
        final int offset = (int) Math.floorMod(position, (long) entrySize);
        position -= offset;
        final byte[] key = state.indexSubspace.pack(groupKey.add(position));
        // This has to be the same size every time, with all the unset bits, or else it gets truncated.
        // We really could use a new mutation that took a linear bit position to set / clear and only did length extension or something like that.
        final byte[] bitmap = new byte[(entrySize + 7) / 8];
        if (remove) {
            if (state.store.isIndexWriteOnly(state.index)) {
                // If the index isn't built, it's possible this key wasn't reached.
                // So initialize it to zeros (or leave it alone).
                state.transaction.mutate(MutationType.BIT_OR, key, bitmap);
            }
            // Otherwise the entry must already exist for us to be removing it,
            // so there is no danger that this will store all (but one) ones in a new key.
            Arrays.fill(bitmap, (byte) 0xFF);
            bitmap[offset / 8] &= ~(byte) (1 << (offset % 8));
            state.transaction.mutate(MutationType.BIT_AND, key, bitmap);
            Arrays.fill(bitmap, (byte) 0x00);
            state.transaction.mutate(MutationType.COMPARE_AND_CLEAR, key, bitmap);
        } else {
            if (unique) {
                // Snapshot read to see if the bit is already set.
                CompletableFuture<Void> future = state.transaction.snapshot().get(key).thenAccept(existing -> {
                    if (existing != null && (existing[offset / 8] & (byte) (1 << (offset % 8))) != 0) {
                        throw new RecordIndexUniquenessViolation(state.index, indexEntry, savedRecord.getPrimaryKey(), // Unfortunately, we don't know the other key.
                        null);
                    }
                });
                futures.add(future);
                // Then arrange to conflict for the position with any concurrent update.
                final byte[] conflictKey = new Subspace(key).pack(offset);
                state.transaction.addReadConflictKey(conflictKey);
                state.transaction.addWriteConflictKey(conflictKey);
            }
            bitmap[offset / 8] |= (byte) (1 << (offset % 8));
            state.transaction.mutate(MutationType.BIT_OR, key, bitmap);
        }
        if (state.store.getTimer() != null) {
            state.store.getTimer().recordSinceNanoTime(FDBStoreTimer.Events.MUTATE_INDEX_ENTRY, startTime);
        }
    }
    return futures != null ? AsyncUtil.whenAll(futures) : AsyncUtil.DONE;
}