Examples with ByteArraySegment io.pravega.common.util.ByteArraySegment used on opensource projects

Example 51 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class DirectMemoryCacheTests method testCacheFull.

/**
 * Tests the ability to notify the caller that a cache is full and handle various situations.
 */
@Test
public void testCacheFull() {
    final int cleanAfterInvocations = DirectMemoryCache.MAX_CLEANUP_ATTEMPTS - 1;
    final int maxSize = LAYOUT.bufferSize();
    final int maxStoredSize = maxSize - LAYOUT.blockSize();
    final BufferView toInsert = new ByteArraySegment(new byte[1]);
    @Cleanup val c = new TestCache(maxSize);
    // Fill up the cache.
    val address1 = c.insert(new ByteArraySegment(new byte[maxStoredSize]));
    checkSnapshot(c, (long) maxStoredSize, (long) maxSize, (long) LAYOUT.blockSize(), (long) maxSize, (long) maxSize);
    // Return value from the setCacheFullCallback supplier.
    val reportClean = new AtomicBoolean(false);
    // If true, will actually perform a cleanup.
    val actualClean = new AtomicBoolean(false);
    // If true, will throw an error.
    val throwError = new AtomicBoolean(false);
    val invocationCount = new AtomicInteger(0);
    c.setCacheFullCallback(() -> {
        invocationCount.incrementAndGet();
        if (actualClean.get() && invocationCount.get() >= cleanAfterInvocations) {
            c.delete(address1);
        }
        if (throwError.get()) {
            throw new IntentionalException();
        }
        return reportClean.get();
    }, 1);
    // 1. No cleanup, no error, reporting the truth. We should try as many times as possible.
    reportClean.set(false);
    actualClean.set(false);
    throwError.set(false);
    invocationCount.set(0);
    AssertExtensions.assertThrows("Expected CacheFullException when no cleanup and no error.", () -> c.insert(toInsert), ex -> ex instanceof CacheFullException);
    Assert.assertEquals("Unexpected number of invocations when no cleanup and no error.", DirectMemoryCache.MAX_CLEANUP_ATTEMPTS, invocationCount.get());
    // 2. No cleanup, no error, reporting that we did clean up. We should try as many times as possible.
    reportClean.set(true);
    actualClean.set(false);
    throwError.set(false);
    invocationCount.set(0);
    AssertExtensions.assertThrows("Expected CacheFullException when no cleanup and no error, but reporting clean.", () -> c.insert(toInsert), ex -> ex instanceof CacheFullException);
    Assert.assertEquals("Unexpected number of invocations when no cleanup but reporting clean.", DirectMemoryCache.MAX_CLEANUP_ATTEMPTS, invocationCount.get());
    // 3. No cleanup, throw error.
    reportClean.set(false);
    actualClean.set(false);
    throwError.set(true);
    invocationCount.set(0);
    AssertExtensions.assertThrows("Expected IntentionalException when error.", () -> c.insert(toInsert), ex -> ex instanceof IntentionalException);
    Assert.assertEquals("Unexpected number of invocations when error.", 1, invocationCount.get());
    // 4. Cleanup.
    reportClean.set(true);
    actualClean.set(true);
    throwError.set(false);
    invocationCount.set(0);
    val address2 = c.insert(toInsert);
    Assert.assertEquals("Unexpected number of invocations when successful cleanup.", cleanAfterInvocations, invocationCount.get());
    Assert.assertEquals("Unexpected second entry.", 1, c.get(address2).getLength());
    checkSnapshot(c, 1L, LAYOUT.blockSize() * 2L, (long) LAYOUT.blockSize(), (long) maxSize, (long) maxSize);
}

Example 52 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class DirectMemoryCacheTests method testRegularOperations.

/**
 * Tests the ability to execute operations in order. We will invoke {@link DirectMemoryCache#insert}, {@link DirectMemoryCache#append},
 * {@link DirectMemoryCache#get} and {@link DirectMemoryCache#delete} without overfilling the cache.
 */
@Test
public void testRegularOperations() {
    final byte[] data = new byte[LAYOUT.bufferSize() * 2];
    final int writeCount = (int) (ACTUAL_MAX_SIZE / LAYOUT.bufferSize()) * (LAYOUT.blocksPerBuffer() - 1);
    rnd.nextBytes(data);
    // Key=Address, Value={Offset, Length}
    val entryData = new HashMap<Integer, Map.Entry<Integer, Integer>>();
    @Cleanup val c = new TestCache();
    // Insert until we can no longer insert.
    CacheState cs = c.getState();
    while ((cs = c.getState()).getUsedBytes() < ACTUAL_MAX_SIZE) {
        int offset = rnd.nextInt(data.length - 1);
        int length = (int) Math.min(cs.getMaxBytes() - cs.getUsedBytes(), rnd.nextInt(data.length - offset));
        int address = c.insert(new ByteArraySegment(data, offset, length));
        entryData.put(address, new AbstractMap.SimpleEntry<>(offset, length));
    }
    // Verify we've actually filled up the cache.
    AssertExtensions.assertThrows("Cache not full.", () -> c.insert(new ByteArraySegment(data, 0, 1)), ex -> ex instanceof CacheFullException);
    // Append to all those entries that we can append to.
    for (val e : entryData.entrySet()) {
        int address = e.getKey();
        int offset = e.getValue().getKey();
        int length = e.getValue().getValue();
        int appendableLength = c.getAppendableLength(length);
        AssertExtensions.assertThrows("append() accepted input that does not fit in last block.", () -> c.append(address, length, new ByteArraySegment(data, 0, appendableLength + 1)), ex -> ex instanceof IllegalArgumentException);
        val a = c.append(address, length, new ByteArraySegment(data, offset + length, appendableLength));
        Assert.assertEquals("Unexpected number of bytes appended.", appendableLength, a);
        e.getValue().setValue(length + a);
    }
    // We expect the cache to be full at this point.
    int reservedBytes = BUFFER_COUNT * LAYOUT.blockSize();
    checkSnapshot(c, ACTUAL_MAX_SIZE - reservedBytes, ACTUAL_MAX_SIZE, (long) reservedBytes, ACTUAL_MAX_SIZE, ACTUAL_MAX_SIZE);
    // Read all the data
    checkData(c, entryData, data);
    // Delete half the entries.
    int targetSize = entryData.size() / 2;
    val deletedData = new HashMap<Integer, Map.Entry<Integer, Integer>>();
    // Keep track of how many blocks we deleted from when we are full - it's easier to track this way.
    int deletedBlockCount = 0;
    for (val e : entryData.entrySet()) {
        int address = e.getKey();
        int offset = e.getValue().getKey();
        int length = e.getValue().getValue();
        deletedBlockCount += length / LAYOUT.blockSize();
        c.delete(address);
        Assert.assertNull("get() returned valid data for deleted entry.", c.get(address));
        deletedData.put(address, new AbstractMap.SimpleEntry<>(offset, length));
        if (deletedData.size() >= targetSize) {
            break;
        }
    }
    deletedData.keySet().forEach(entryData::remove);
    // Verify the cache has been freed of all of them.
    int deletedBytes = deletedBlockCount * LAYOUT.blockSize();
    checkSnapshot(c, ACTUAL_MAX_SIZE - reservedBytes - deletedBytes, ACTUAL_MAX_SIZE - deletedBytes, (long) reservedBytes, ACTUAL_MAX_SIZE, ACTUAL_MAX_SIZE);
    // Replace all existing entries with the ones we just deleted.
    val toReplace = new ArrayDeque<>(entryData.keySet());
    val replacements = new ArrayDeque<>(deletedData.values());
    while (!toReplace.isEmpty() && !replacements.isEmpty()) {
        val address = toReplace.removeFirst();
        val replacement = replacements.removeFirst();
        val newAddress = c.replace(address, new ByteArraySegment(data, replacement.getKey(), replacement.getValue()));
        val r = entryData.remove(address);
        // Account for this entry's removal.
        deletedBlockCount += r.getValue() / LAYOUT.blockSize();
        entryData.put(newAddress, replacement);
        // Account for this entry's replacement.
        deletedBlockCount -= replacement.getValue() / LAYOUT.blockSize();
    }
    // Verify final data.
    checkData(c, entryData, data);
    // Verify the cache has as many bytes as we think it has.
    deletedBytes = deletedBlockCount * LAYOUT.blockSize();
    checkSnapshot(c, ACTUAL_MAX_SIZE - reservedBytes - deletedBytes, ACTUAL_MAX_SIZE - deletedBytes, (long) reservedBytes, ACTUAL_MAX_SIZE, ACTUAL_MAX_SIZE);
    // Delete the rest and verify everything has been cleared.
    entryData.keySet().forEach(c::delete);
    checkSnapshot(c, 0L, (long) reservedBytes, (long) reservedBytes, ACTUAL_MAX_SIZE, ACTUAL_MAX_SIZE);
}

Example 53 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class DirectMemoryCacheTests method testAllocateClose.

/**
 * Verifies the allocation and deallocation (when closing) of direct memory.
 */
@Test
public void testAllocateClose() {
    final int writeSize = 1;
    long storedBytes = 0;
    long usedBytes = 0;
    long reservedBytes = 0;
    long allocatedBytes = 0;
    // Initially it should be empty.
    @Cleanup val c = new TestCache();
    checkSnapshot(c, storedBytes, usedBytes, reservedBytes, allocatedBytes, ACTUAL_MAX_SIZE);
    // Fill up the cache and verify, at each step, what should happen. Pre-calculate the number of writes that would
    // fill it up so we would know right away if something gets missed or overwritten.
    int writeCount = BUFFER_COUNT * (LAYOUT.blocksPerBuffer() - 1);
    int lastBufferId = -1;
    for (int i = 0; i < writeCount; i++) {
        val address = c.insert(new ByteArraySegment(new byte[writeSize]));
        int bufferId = LAYOUT.getBufferId(address);
        if (bufferId != lastBufferId) {
            lastBufferId = bufferId;
            allocatedBytes += LAYOUT.bufferSize();
            // metadata block.
            reservedBytes += LAYOUT.blockSize();
            usedBytes += LAYOUT.blockSize();
        }
        storedBytes += writeSize;
        usedBytes += LAYOUT.blockSize();
        checkSnapshot(c, storedBytes, usedBytes, reservedBytes, allocatedBytes, ACTUAL_MAX_SIZE);
    }
    AssertExtensions.assertThrows("Expecting cache to be full.", () -> c.insert(new ByteArraySegment(new byte[1])), ex -> ex instanceof CacheFullException);
// Invoking close() (@Cleanup) will also verify it freed all the memory.
}

Example 54 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class TableBasedMetadataStore method writeAll.

/**
 * Writes transaction data from a given list to the metadata store.
 *
 * @param dataList List of transaction data to write.
 */
@Override
protected CompletableFuture<Void> writeAll(Collection<TransactionData> dataList) {
    val toUpdate = new ArrayList<TableEntry>();
    val entryToTxnDataMap = new HashMap<TableEntry, TransactionData>();
    val deletedKeyToTxnDataMap = new HashMap<TableKey, TransactionData>();
    val keysToDelete = new ArrayList<TableKey>();
    val t = new Timer();
    return ensureInitialized().thenRunAsync(() -> {
        for (TransactionData txnData : dataList) {
            Preconditions.checkState(null != txnData.getDbObject(), "Missing tracking object");
            val version = (Long) txnData.getDbObject();
            if (null == txnData.getValue()) {
                val toDelete = TableKey.unversioned(new ByteArraySegment(txnData.getKey().getBytes(Charsets.UTF_8)));
                keysToDelete.add(toDelete);
                deletedKeyToTxnDataMap.put(toDelete, txnData);
            }
            try {
                val arraySegment = SERIALIZER.serialize(txnData);
                TableEntry tableEntry = TableEntry.versioned(new ByteArraySegment(txnData.getKey().getBytes(Charsets.UTF_8)), arraySegment, version);
                entryToTxnDataMap.put(tableEntry, txnData);
                toUpdate.add(tableEntry);
            } catch (Exception e) {
                throw new CompletionException(handleException(e));
            }
        }
    }, getExecutor()).thenComposeAsync(v -> {
        // toUpdate includes both modified keys as well updates to deleted keys to mark them as deleted.
        return this.tableStore.put(tableName, toUpdate, timeout).thenComposeAsync(ret -> {
            // Update versions.
            int i = 0;
            for (TableEntry tableEntry : toUpdate) {
                entryToTxnDataMap.get(tableEntry).setDbObject(ret.get(i));
                i++;
            }
            // This next step will just remove them from table store.
            return this.tableStore.remove(tableName, keysToDelete, timeout).handleAsync((v1, ex) -> {
                // keys are already persisted successfully in earlier step.
                if (ex == null) {
                    deletedKeyToTxnDataMap.values().stream().forEach(txnData -> txnData.setDbObject(TableKey.NOT_EXISTS));
                } else {
                    log.warn("Error while deleting keys from table segment {}.", tableName, ex);
                }
                TABLE_WRITE_LATENCY.reportSuccessEvent(t.getElapsed());
                return v1;
            }, getExecutor());
        }, getExecutor());
    }, getExecutor()).exceptionally(e -> {
        val ex = Exceptions.unwrap(e);
        throw new CompletionException(handleException(ex));
    });
}

Example 55 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class TableBasedMetadataStore method read.

/**
 * Reads a metadata record for the given key.
 *
 * @param key Key for the metadata record.
 * @return Associated {@link io.pravega.segmentstore.storage.metadata.BaseMetadataStore.TransactionData}.
 */
@Override
protected CompletableFuture<TransactionData> read(String key) {
    val keys = new ArrayList<BufferView>();
    keys.add(new ByteArraySegment(key.getBytes(Charsets.UTF_8)));
    val t = new Timer();
    return ensureInitialized().thenComposeAsync(v -> this.tableStore.get(tableName, keys, timeout).thenApplyAsync(entries -> {
        try {
            Preconditions.checkState(entries.size() == 1, "Unexpected number of values returned.");
            val entry = entries.get(0);
            if (null != entry) {
                val arr = entry.getValue();
                TransactionData txnData = SERIALIZER.deserialize(arr);
                txnData.setDbObject(entry.getKey().getVersion());
                txnData.setPersisted(true);
                TABLE_GET_LATENCY.reportSuccessEvent(t.getElapsed());
                METADATA_FOUND_IN_STORE.inc();
                return txnData;
            }
        } catch (Exception e) {
            throw new CompletionException(new StorageMetadataException("Error while reading", e));
        }
        TABLE_GET_LATENCY.reportSuccessEvent(t.getElapsed());
        METADATA_NOT_FOUND.inc();
        return TransactionData.builder().key(key).persisted(true).dbObject(TableKey.NOT_EXISTS).build();
    }, getExecutor()).exceptionally(e -> {
        val ex = Exceptions.unwrap(e);
        throw new CompletionException(handleException(ex));
    }), getExecutor());
}