Examples with BufferView io.pravega.common.util.BufferView used on opensource projects

Example 56 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class AsyncReadResultProcessorTests method testProcessAll.

/**
 * Tests the {@link AsyncReadResultProcessor#processAll} method.
 */
@Test
public void testProcessAll() throws Exception {
    // Pre-generate some entries.
    ArrayList<byte[]> entries = new ArrayList<>();
    int totalLength = generateEntries(entries);
    // Setup an entry provider supplier.
    AtomicInteger currentIndex = new AtomicInteger();
    StreamSegmentReadResult.NextEntrySupplier supplier = (offset, length, makeCopy) -> {
        int idx = currentIndex.getAndIncrement();
        if (idx == entries.size() - 1) {
            // Future read result.
            Supplier<BufferView> entryContentsSupplier = () -> new ByteArraySegment(entries.get(idx));
            return new TestFutureReadResultEntry(offset, length, entryContentsSupplier, executorService());
        } else if (idx >= entries.size()) {
            return null;
        }
        // Normal read.
        return new CacheReadResultEntry(offset, entries.get(idx), 0, entries.get(idx).length);
    };
    // Fetch all the data and compare with expected.
    @Cleanup StreamSegmentReadResult rr = new StreamSegmentReadResult(0, totalLength, supplier, "");
    val result = AsyncReadResultProcessor.processAll(rr, executorService(), TIMEOUT);
    val actualData = result.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).getReader();
    val expectedData = new SequenceInputStream(Iterators.asEnumeration(entries.stream().map(ByteArrayInputStream::new).iterator()));
    AssertExtensions.assertStreamEquals("Unexpected data read back.", expectedData, actualData, totalLength);
}

Example 57 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class DirectMemoryCache method insert.

@Override
public int insert(BufferView data) {
    Exceptions.checkNotClosed(this.closed.get(), this);
    Preconditions.checkArgument(data.getLength() <= CacheLayout.MAX_ENTRY_SIZE, "Entry too long. Expected max %s, given %s.", CacheLayout.MAX_ENTRY_SIZE, data.getLength());
    int lastBlockAddress = CacheLayout.NO_ADDRESS;
    int remainingLength = data.getLength();
    try {
        // and write the remaining data to it.
        while (remainingLength > 0 || lastBlockAddress == CacheLayout.NO_ADDRESS) {
            // Get a Buffer to write data to. If we are full, this will throw an appropriate exception.
            DirectMemoryBuffer buffer = getNextAvailableBuffer();
            // Write the data to the buffer.
            BufferView slice = data.slice(data.getLength() - remainingLength, remainingLength);
            DirectMemoryBuffer.WriteResult writeResult = buffer.write(slice, lastBlockAddress);
            if (writeResult == null) {
                // Someone else grabbed this buffer and wrote to it before we got a chance. Go back and find another one.
                continue;
            }
            // invoking delete() will undo this changes as well.
            assert writeResult.getWrittenLength() >= 0 && writeResult.getWrittenLength() <= remainingLength : writeResult.getWrittenLength();
            remainingLength -= writeResult.getWrittenLength();
            this.storedBytes.addAndGet(writeResult.getWrittenLength());
            lastBlockAddress = writeResult.getLastBlockAddress();
        }
    } catch (Throwable ex) {
        if (!Exceptions.mustRethrow(ex) && lastBlockAddress != CacheLayout.NO_ADDRESS) {
            // We wrote something, but got interrupted. We need to clean up whatever we wrote so we don't leave
            // unreferenced data in the cache.
            delete(lastBlockAddress);
        }
        throw ex;
    }
    this.metrics.insert(data.getLength());
    return lastBlockAddress;
}

Example 58 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class FixedKeyLengthTableSegmentLayout method get.

@Override
CompletableFuture<List<TableEntry>> get(@NonNull DirectSegmentAccess segment, @NonNull List<BufferView> keys, TimeoutTimer timer) {
    val segmentInfo = segment.getInfo();
    ensureSegmentType(segmentInfo.getName(), segmentInfo.getType());
    val attributes = keys.stream().map(key -> {
        ensureValidKeyLength(segmentInfo.getName(), key.getLength());
        return AttributeId.from(key.getCopy());
    }).collect(Collectors.toList());
    logRequest("get", segmentInfo.getName(), keys.size());
    return getByAttributes(segment, attributes, timer);
}

Example 59 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class SegmentAggregator method getFlushData.

/**
 * Returns a {@link BufferView} which contains the data needing to be flushed to Storage.
 *
 * @return A {@link BufferView} to flush or null if the segment was deleted.
 * @throws DataCorruptionException If a unable to retrieve required data from the Data Source.
 */
@Nullable
private BufferView getFlushData() throws DataCorruptionException {
    StorageOperation first = this.operations.getFirst();
    if (!(first instanceof AggregatedAppendOperation)) {
        // Nothing to flush - first operation is not an AggregatedAppend.
        return null;
    }
    AggregatedAppendOperation appendOp = (AggregatedAppendOperation) first;
    int length = (int) appendOp.getLength();
    BufferView data = null;
    if (length > 0) {
        data = this.dataSource.getAppendData(appendOp.getStreamSegmentId(), appendOp.getStreamSegmentOffset(), length);
        if (data == null) {
            if (this.metadata.isDeleted()) {
                // Segment was deleted - nothing more to do.
                return null;
            }
            throw new DataCorruptionException(String.format("Unable to retrieve CacheContents for '%s'.", appendOp));
        }
    }
    appendOp.seal();
    return data;
}

Example 60 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class SegmentAggregator method reconcileData.

/**
 * Attempts to reconcile the data for the given AggregatedAppendOperation. Since Append Operations can be partially
 * flushed, reconciliation may be for the full operation or for a part of it.
 *
 * @param op          The AggregatedAppendOperation to reconcile.
 * @param storageInfo The current state of the Segment in Storage.
 * @param timer       Timer for the operation.
 * @return A CompletableFuture containing a FlushResult with the number of bytes reconciled, or failed with a ReconciliationFailureException,
 * if the operation cannot be reconciled, based on the in-memory metadata or the current state of the Segment in Storage.
 */
private CompletableFuture<Integer> reconcileData(AggregatedAppendOperation op, SegmentProperties storageInfo, TimeoutTimer timer) {
    BufferView appendData = this.dataSource.getAppendData(op.getStreamSegmentId(), op.getStreamSegmentOffset(), (int) op.getLength());
    if (appendData == null) {
        return Futures.failedFuture(new ReconciliationFailureException(String.format("Unable to reconcile operation '%s' because no append data is associated with it.", op), this.metadata, storageInfo));
    }
    // Only read as much data as we need.
    long readLength = Math.min(op.getLastStreamSegmentOffset(), storageInfo.getLength()) - op.getStreamSegmentOffset();
    assert readLength > 0 : "Append Operation to be reconciled is beyond the Segment's StorageLength (" + storageInfo.getLength() + "): " + op;
    // Read all data from storage.
    byte[] storageData = new byte[(int) readLength];
    AtomicInteger reconciledBytes = new AtomicInteger();
    return Futures.loop(() -> reconciledBytes.get() < readLength, () -> this.storage.read(this.handle.get(), op.getStreamSegmentOffset() + reconciledBytes.get(), storageData, reconciledBytes.get(), (int) readLength - reconciledBytes.get(), timer.getRemaining()), bytesRead -> {
        assert bytesRead > 0 : String.format("Unable to make any read progress when reconciling operation '%s' after reading %s bytes.", op, reconciledBytes);
        reconciledBytes.addAndGet(bytesRead);
    }, this.executor).thenApplyAsync(v -> {
        // Compare, byte-by-byte, the contents of the append.
        verifySame(appendData, storageData, op, storageInfo);
        return reconciledBytes.get();
    }, this.executor);
}