Examples with TimeoutTimer io.pravega.common.TimeoutTimer used on opensource projects

Example 46 with TimeoutTimer

use of io.pravega.common.TimeoutTimer in project pravega by pravega.

the class IndexWriter method groupByBucket.

// endregion
// region Updating Table Buckets
/**
 * Groups the given {@link BucketUpdate.KeyUpdate} instances by their associated buckets.
 *
 * @param segment    The Segment to read from.
 * @param keyUpdates A Collection of {@link BucketUpdate.KeyUpdate} instances to index.
 * @param timer      Timer for the operation.
 * @return A CompletableFuture that, when completed, will contain the a collection of {@link BucketUpdate.Builder}s.
 */
CompletableFuture<Collection<BucketUpdate.Builder>> groupByBucket(DirectSegmentAccess segment, Collection<BucketUpdate.KeyUpdate> keyUpdates, TimeoutTimer timer) {
    val updatesByHash = keyUpdates.stream().collect(Collectors.groupingBy(k -> this.hasher.hash(k.getKey())));
    return locateBuckets(segment, updatesByHash.keySet(), timer).thenApplyAsync(buckets -> {
        val result = new HashMap<TableBucket, BucketUpdate.Builder>();
        buckets.forEach((keyHash, bucket) -> {
            // Add the bucket to the result and record this Key as a "new" key in it.
            BucketUpdate.Builder bu = result.computeIfAbsent(bucket, BucketUpdate::forBucket);
            updatesByHash.get(keyHash).forEach(bu::withKeyUpdate);
        });
        return result.values();
    }, this.executor);
}

Example 47 with TimeoutTimer

use of io.pravega.common.TimeoutTimer in project pravega by pravega.

the class WriterTableProcessor method flushOnce.

/**
 * Performs a single flush attempt.
 *
 * @param segment A {@link DirectSegmentAccess} representing the Segment to flush on.
 * @param timer   Timer for the operation.
 * @return A CompletableFuture that, when completed, will indicate the flush has completed successfully. If the
 * operation failed, it will be failed with the appropriate exception. Notable exceptions:
 * <ul>
 * <li>{@link BadAttributeUpdateException} If a conditional update on the {@link TableAttributes#INDEX_OFFSET} attribute failed.
 * </ul>
 */
private CompletableFuture<TableWriterFlushResult> flushOnce(DirectSegmentAccess segment, TimeoutTimer timer) {
    // Index all the keys in the segment range pointed to by the aggregator.
    long lastOffset = this.aggregator.getLastIndexToProcessAtOnce(this.connector.getMaxFlushSize());
    assert lastOffset - this.aggregator.getFirstOffset() <= this.connector.getMaxFlushSize();
    if (lastOffset < this.aggregator.getLastOffset()) {
        log.info("{}: Partial flush initiated up to offset {}. State: {}.", this.traceObjectId, lastOffset, this.aggregator);
    }
    KeyUpdateCollection keyUpdates = readKeysFromSegment(segment, this.aggregator.getFirstOffset(), lastOffset, timer);
    log.debug("{}: Flush.ReadFromSegment KeyCount={}, UpdateCount={}, HighestCopiedOffset={}, LastIndexedOffset={}.", this.traceObjectId, keyUpdates.getUpdates().size(), keyUpdates.getTotalUpdateCount(), keyUpdates.getHighestCopiedOffset(), keyUpdates.getLastIndexedOffset());
    // for each such bucket and finally (reindex) update the bucket.
    return this.indexWriter.groupByBucket(segment, keyUpdates.getUpdates(), timer).thenComposeAsync(builders -> fetchExistingKeys(builders, segment, timer).thenComposeAsync(v -> {
        val bucketUpdates = builders.stream().map(BucketUpdate.Builder::build).collect(Collectors.toList());
        logBucketUpdates(bucketUpdates);
        return this.indexWriter.updateBuckets(segment, bucketUpdates, this.aggregator.getLastIndexedOffset(), keyUpdates.getLastIndexedOffset(), keyUpdates.getTotalUpdateCount(), timer.getRemaining());
    }, this.executor), this.executor).thenApply(updateCount -> new TableWriterFlushResult(keyUpdates, updateCount));
}

Example 48 with TimeoutTimer

use of io.pravega.common.TimeoutTimer in project pravega by pravega.

the class TableBucketReader method findAllExisting.

// endregion
// region Searching
/**
 * Locates all {@link ResultT} instances in a TableBucket.
 *
 * @param bucketOffset The current segment offset of the Table Bucket we are looking into.
 * @param timer        A {@link TimeoutTimer} for the operation.
 * @return A CompletableFuture that, when completed, will contain a List with the desired result items. This list
 * will exclude all {@link ResultT} items that are marked as deleted.
 */
CompletableFuture<List<ResultT>> findAllExisting(long bucketOffset, TimeoutTimer timer) {
    val result = new HashMap<BufferView, ResultT>();
    // This handler ensures that items are only added once (per key) and only if they are not deleted. Since the items
    // are processed in descending version order, the first time we encounter its key is its latest value.
    BiConsumer<ResultT, Long> handler = (item, offset) -> {
        TableKey key = getKey(item);
        if (!result.containsKey(key.getKey())) {
            result.put(key.getKey(), key.getVersion() == TableKey.NOT_EXISTS ? null : item);
        }
    };
    return findAll(bucketOffset, handler, timer).thenApply(v -> result.values().stream().filter(Objects::nonNull).collect(Collectors.toList()));
}

Example 49 with TimeoutTimer

use of io.pravega.common.TimeoutTimer in project pravega by pravega.

the class HashTableCompactor method excludeObsolete.

/**
 * Processes the given {@link CompactionArgs} and eliminates all {@link Candidate}s that meet at least one of the
 * following criteria:
 * - The Key's Table Bucket is no longer part of the index (removal)
 * - The Key exists in the Index, but the Index points to a newer version of it.
 *
 * @param args    A {@link CompactionArgs} representing the set of {@link Candidate}s for compaction. This set
 *                will be modified based on the outcome of this method.
 * @param buckets The Buckets retrieved via the {@link IndexReader} for the {@link Candidate}s.
 * @param timer   Timer for the operation.
 * @return A CompletableFuture that, when completed, will indicate the operation has finished.
 */
private CompletableFuture<Void> excludeObsolete(IndexedCompactionArgs args, Map<UUID, TableBucket> buckets, TimeoutTimer timer) {
    // Exclude all those Table Entries whose buckets altogether do not exist.
    val deletedBuckets = args.candidatesByHash.keySet().stream().filter(k -> {
        val bucket = buckets.get(k);
        return bucket == null || !bucket.exists();
    }).collect(Collectors.toList());
    // Do this in a separate loop since we are modifying args.candidatesByHash with removeBucket().
    for (val bucket : deletedBuckets) {
        args.removeBucket(bucket);
    }
    // For every Bucket that still exists, find all its Keys and match with our candidates and figure out if our
    // candidates are still eligible for compaction.
    val br = TableBucketReader.key(this.segment, this.indexReader::getBackpointerOffset, this.executor);
    val candidateBuckets = args.candidatesByHash.keySet().iterator();
    return Futures.loop(candidateBuckets::hasNext, () -> {
        val bucketId = candidateBuckets.next();
        long bucketOffset = buckets.get(bucketId).getSegmentOffset();
        return br.findAll(bucketOffset, args::handleExistingKey, timer);
    }, this.executor);
}

Example 50 with TimeoutTimer

use of io.pravega.common.TimeoutTimer in project pravega by pravega.

the class StreamSegmentContainerTests method waitForOperationsInReadIndex.

/**
 * Blocks until all operations processed so far have been added to the ReadIndex and InMemoryOperationLog.
 * This is needed to simplify test verification due to the fact that the the OperationProcessor commits operations to
 * the ReadIndex and InMemoryOperationLog asynchronously, after those operations were ack-ed. This method makes use
 * of the fact that the OperationProcessor/MemoryStateUpdater will still commit such operations in sequence; it
 * creates a new segment, writes 1 byte to it and issues a read (actual/future) and waits until it's completed - when
 * it is, it is guaranteed that everything prior to that has been committed.
 */
private static void waitForOperationsInReadIndex(SegmentContainer container) throws Exception {
    TimeoutTimer timer = new TimeoutTimer(TIMEOUT);
    String segmentName = "test" + System.nanoTime();
    container.createStreamSegment(segmentName, BASIC_TYPE, null, timer.getRemaining()).thenCompose(v -> container.append(segmentName, new ByteArraySegment(new byte[1]), null, timer.getRemaining())).thenCompose(v -> container.read(segmentName, 0, 1, timer.getRemaining())).thenCompose(rr -> {
        ReadResultEntry rre = rr.next();
        rre.requestContent(TIMEOUT);
        return rre.getContent().thenRun(rr::close);
    }).thenCompose(v -> container.deleteStreamSegment(segmentName, timer.getRemaining())).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
}