Examples with DirectSegmentAccess io.pravega.segmentstore.server.DirectSegmentAccess used on opensource projects

Example 1 with DirectSegmentAccess

use of io.pravega.segmentstore.server.DirectSegmentAccess in project pravega by pravega.

the class ContainerKeyIndex method validateConditionalUpdateFailures.

private CompletableFuture<Void> validateConditionalUpdateFailures(DirectSegmentAccess segment, Map<TableKey, Long> expectedVersions, TimeoutTimer timer) {
    assert !expectedVersions.isEmpty();
    val bucketReader = TableBucketReader.key(segment, this::getBackpointerOffset, this.executor);
    val searches = expectedVersions.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> findBucketEntry(segment, bucketReader, e.getKey().getKey(), e.getValue(), timer)));
    return Futures.allOf(searches.values()).thenRun(() -> {
        val failed = new HashMap<TableKey, Long>();
        for (val e : searches.entrySet()) {
            val actual = e.getValue().join();
            boolean isValid = actual == null ? e.getKey().getVersion() == TableKey.NOT_EXISTS : e.getKey().getVersion() == actual.getVersion();
            if (!isValid) {
                failed.put(e.getKey(), actual == null ? TableKey.NOT_EXISTS : actual.getVersion());
            }
        }
        if (!failed.isEmpty()) {
            throw new CompletionException(new BadKeyVersionException(segment.getInfo().getName(), failed));
        }
    });
}

Example 2 with DirectSegmentAccess

use of io.pravega.segmentstore.server.DirectSegmentAccess in project pravega by pravega.

the class ContainerKeyIndex method update.

/**
 * Performs a Batch Update or Removal.
 *
 * If {@link TableKeyBatch#isConditional()} returns true, this will execute an atomic Conditional Update/Removal based
 * on the condition items in the batch ({@link TableKeyBatch#getVersionedItems}. The entire TableKeyBatch will be
 * conditioned on those items, including those items that do not have a condition set. The entire TableKeyBatch will
 * either all be committed as one unit or not at all.
 *
 * Otherwise this will perform an Unconditional Update/Removal, where all the {@link TableKeyBatch.Item}s in
 * {@link TableKeyBatch#getItems()} will be applied regardless of whether they already exist or what their versions are.
 *
 * @param segment The Segment to perform the update/removal on.
 * @param batch   The {@link TableKeyBatch} to apply.
 * @param persist A Supplier that, when invoked, will persist the contents of the batch to the Segment and return
 *                a CompletableFuture to indicate when the operation is done, containing the offset at which the
 *                batch has been written to the Segment. This Future must complete successfully before the effects
 *                of the {@link TableKeyBatch} are applied to the in-memory Index or before downstream conditional
 *                updates on the affected keys are initiated.
 * @param timer   Timer for the operation.
 * @return A CompletableFuture that, when completed, will contain a list of offsets (within the Segment) where each
 * of the items in the batch has been persisted. If the update failed, it will be failed with the appropriate exception.
 * Notable exceptions:
 * <ul>
 * <li>{@link KeyNotExistsException} If a Key in the TableKeyBatch does not exist and was conditioned as having to exist.
 * <li>{@link BadKeyVersionException} If a Key does exist but had a version mismatch.
 * </ul>
 */
CompletableFuture<List<Long>> update(DirectSegmentAccess segment, TableKeyBatch batch, Supplier<CompletableFuture<Long>> persist, TimeoutTimer timer) {
    Exceptions.checkNotClosed(this.closed.get(), this);
    Supplier<CompletableFuture<List<Long>>> update;
    if (batch.isConditional()) {
        // Conditional update.
        // Collect all Cache Keys for the Update Items that have a condition on them; we need this on order to
        // serialize execution across them.
        val keys = batch.getVersionedItems().stream().map(item -> Maps.immutableEntry(segment.getSegmentId(), item.getHash())).collect(Collectors.toList());
        // Serialize the execution (queue it up to run only after all other currently queued up conditional updates
        // for touched keys have finished).
        update = () -> this.conditionalUpdateProcessor.add(keys, () -> validateConditionalUpdate(segment, batch, timer).thenComposeAsync(v -> persist.get(), this.executor).thenApplyAsync(batchOffset -> updateCache(segment, batch, batchOffset), this.executor));
    } else {
        // Unconditional update: persist the entries and update the cache.
        update = () -> persist.get().thenApplyAsync(batchOffset -> updateCache(segment, batch, batchOffset), this.executor);
    }
    // Throttle any requests, if needed.
    return this.segmentTracker.throttleIfNeeded(segment, update, batch.getLength());
}

Example 3 with DirectSegmentAccess

use of io.pravega.segmentstore.server.DirectSegmentAccess in project pravega by pravega.

the class HashTableSegmentLayout method newIterator.

private <T> CompletableFuture<AsyncIterator<IteratorItem<T>>> newIterator(@NonNull DirectSegmentAccess segment, @NonNull IteratorArgs args, @NonNull GetBucketReader<T> createBucketReader) {
    Preconditions.checkArgument(args.getFrom() == null && args.getTo() == null, "Range Iterators not supported for HashTableSegments.");
    UUID fromHash;
    BufferView serializedState = args.getContinuationToken();
    try {
        fromHash = KeyHasher.getNextHash(serializedState == null ? null : IteratorStateImpl.deserialize(serializedState).getKeyHash());
    } catch (IOException ex) {
        // Bad IteratorState serialization.
        throw new IllegalDataFormatException("Unable to deserialize `serializedState`.", ex);
    }
    if (fromHash == null) {
        // Nothing to iterate on.
        return CompletableFuture.completedFuture(TableIterator.empty());
    }
    // Create a converter that will use a TableBucketReader to fetch all requested items in the iterated Buckets.
    val bucketReader = createBucketReader.apply(segment, this.keyIndex::getBackpointerOffset, this.executor);
    TableIterator.ConvertResult<IteratorItem<T>> converter = bucket -> bucketReader.findAllExisting(bucket.getSegmentOffset(), new TimeoutTimer(args.getFetchTimeout())).thenApply(result -> new IteratorItemImpl<>(new IteratorStateImpl(bucket.getHash()).serialize(), result));
    // Fetch the Tail (Unindexed) Hashes, then create the TableIterator.
    return this.keyIndex.getUnindexedKeyHashes(segment).thenComposeAsync(cacheHashes -> TableIterator.<IteratorItem<T>>builder().segment(segment).cacheHashes(cacheHashes).firstHash(fromHash).executor(executor).resultConverter(converter).fetchTimeout(args.getFetchTimeout()).build(), this.executor);
}

Example 4 with DirectSegmentAccess

use of io.pravega.segmentstore.server.DirectSegmentAccess in project pravega by pravega.

the class IndexWriter method groupByBucket.

// endregion
// region Updating Table Buckets
/**
 * Groups the given {@link BucketUpdate.KeyUpdate} instances by their associated buckets.
 *
 * @param segment    The Segment to read from.
 * @param keyUpdates A Collection of {@link BucketUpdate.KeyUpdate} instances to index.
 * @param timer      Timer for the operation.
 * @return A CompletableFuture that, when completed, will contain the a collection of {@link BucketUpdate.Builder}s.
 */
CompletableFuture<Collection<BucketUpdate.Builder>> groupByBucket(DirectSegmentAccess segment, Collection<BucketUpdate.KeyUpdate> keyUpdates, TimeoutTimer timer) {
    val updatesByHash = keyUpdates.stream().collect(Collectors.groupingBy(k -> this.hasher.hash(k.getKey())));
    return locateBuckets(segment, updatesByHash.keySet(), timer).thenApplyAsync(buckets -> {
        val result = new HashMap<TableBucket, BucketUpdate.Builder>();
        buckets.forEach((keyHash, bucket) -> {
            // Add the bucket to the result and record this Key as a "new" key in it.
            BucketUpdate.Builder bu = result.computeIfAbsent(bucket, BucketUpdate::forBucket);
            updatesByHash.get(keyHash).forEach(bu::withKeyUpdate);
        });
        return result.values();
    }, this.executor);
}

Example 5 with DirectSegmentAccess

use of io.pravega.segmentstore.server.DirectSegmentAccess in project pravega by pravega.

the class WriterTableProcessor method flushOnce.

/**
 * Performs a single flush attempt.
 *
 * @param segment A {@link DirectSegmentAccess} representing the Segment to flush on.
 * @param timer   Timer for the operation.
 * @return A CompletableFuture that, when completed, will indicate the flush has completed successfully. If the
 * operation failed, it will be failed with the appropriate exception. Notable exceptions:
 * <ul>
 * <li>{@link BadAttributeUpdateException} If a conditional update on the {@link TableAttributes#INDEX_OFFSET} attribute failed.
 * </ul>
 */
private CompletableFuture<TableWriterFlushResult> flushOnce(DirectSegmentAccess segment, TimeoutTimer timer) {
    // Index all the keys in the segment range pointed to by the aggregator.
    long lastOffset = this.aggregator.getLastIndexToProcessAtOnce(this.connector.getMaxFlushSize());
    assert lastOffset - this.aggregator.getFirstOffset() <= this.connector.getMaxFlushSize();
    if (lastOffset < this.aggregator.getLastOffset()) {
        log.info("{}: Partial flush initiated up to offset {}. State: {}.", this.traceObjectId, lastOffset, this.aggregator);
    }
    KeyUpdateCollection keyUpdates = readKeysFromSegment(segment, this.aggregator.getFirstOffset(), lastOffset, timer);
    log.debug("{}: Flush.ReadFromSegment KeyCount={}, UpdateCount={}, HighestCopiedOffset={}, LastIndexedOffset={}.", this.traceObjectId, keyUpdates.getUpdates().size(), keyUpdates.getTotalUpdateCount(), keyUpdates.getHighestCopiedOffset(), keyUpdates.getLastIndexedOffset());
    // for each such bucket and finally (reindex) update the bucket.
    return this.indexWriter.groupByBucket(segment, keyUpdates.getUpdates(), timer).thenComposeAsync(builders -> fetchExistingKeys(builders, segment, timer).thenComposeAsync(v -> {
        val bucketUpdates = builders.stream().map(BucketUpdate.Builder::build).collect(Collectors.toList());
        logBucketUpdates(bucketUpdates);
        return this.indexWriter.updateBuckets(segment, bucketUpdates, this.aggregator.getLastIndexedOffset(), keyUpdates.getLastIndexedOffset(), keyUpdates.getTotalUpdateCount(), timer.getRemaining());
    }, this.executor), this.executor).thenApply(updateCount -> new TableWriterFlushResult(keyUpdates, updateCount));
}