Examples with CompositeByteArraySegment io.pravega.common.util.CompositeByteArraySegment used on opensource projects

Example 6 with CompositeByteArraySegment

use of io.pravega.common.util.CompositeByteArraySegment in project pravega by pravega.

the class BookKeeperLogTests method testMissingEmptyLedgers.

/**
 * Verifies that {@link BookKeeperLog#initialize} is able to handle the situation when a Ledger is marked as Empty
 * but it is also deleted from BookKeeper. This ledger should be ignored and not cause the initialization to fail.
 */
@Test
public void testMissingEmptyLedgers() throws Exception {
    final int count = 10;
    // Every 10th Ledger has data.
    final int writeEvery = 5;
    final Predicate<Integer> shouldAppendAnything = i -> i % writeEvery == 0;
    val currentMetadata = new AtomicReference<LogMetadata>();
    for (int i = 0; i < count; i++) {
        boolean isEmpty = !shouldAppendAnything.test(i);
        // boolean isDeleted = shouldDelete.test(i);
        try (BookKeeperLog log = (BookKeeperLog) createDurableDataLog()) {
            log.initialize(TIMEOUT);
            currentMetadata.set(log.loadMetadata());
            // Delete the last Empty ledger, if any.
            val toDelete = Lists.reverse(currentMetadata.get().getLedgers()).stream().filter(m -> m.getStatus() == LedgerMetadata.Status.Empty).findFirst().orElse(null);
            if (toDelete != null) {
                Ledgers.delete(toDelete.getLedgerId(), this.factory.get().getBookKeeperClient());
            }
            // Append some data to this Ledger, if needed - this will mark it as NotEmpty.
            if (!isEmpty) {
                log.append(new CompositeByteArraySegment(getWriteData()), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            }
        }
    }
    // Now try to delete a valid ledger and verify that an exception was actually thrown.
    val validLedgerToDelete = Lists.reverse(currentMetadata.get().getLedgers()).stream().filter(m -> m.getStatus() != LedgerMetadata.Status.Empty).findFirst().orElse(null);
    if (validLedgerToDelete != null) {
        Ledgers.delete(validLedgerToDelete.getLedgerId(), this.factory.get().getBookKeeperClient());
    }
    AssertExtensions.assertThrows("No exception thrown if valid ledger was deleted.", () -> {
        @Cleanup val log = createDurableDataLog();
        log.initialize(TIMEOUT);
    }, ex -> ex instanceof DurableDataLogException && ex.getCause() instanceof BKException.BKNoSuchLedgerExistsOnMetadataServerException);
}

Example 7 with CompositeByteArraySegment

use of io.pravega.common.util.CompositeByteArraySegment in project pravega by pravega.

the class BookKeeperLogTests method testAppendTransientBookieFailure.

/**
 * Tests the ability to retry writes when Bookies fail.
 */
@Test
public void testAppendTransientBookieFailure() throws Exception {
    TreeMap<LogAddress, byte[]> writeData = new TreeMap<>(Comparator.comparingLong(LogAddress::getSequence));
    try (DurableDataLog log = createDurableDataLog()) {
        log.initialize(TIMEOUT);
        val dataList = new ArrayList<byte[]>();
        val futures = new ArrayList<CompletableFuture<LogAddress>>();
        try {
            // Suspend a bookie (this will trigger write errors).
            suspendFirstBookie();
            // Issue appends in parallel, without waiting for them.
            int writeCount = getWriteCount();
            for (int i = 0; i < writeCount; i++) {
                byte[] data = getWriteData();
                futures.add(log.append(new CompositeByteArraySegment(data), TIMEOUT));
                dataList.add(data);
            }
        } finally {
            // Resume the bookie with the appends still in flight.
            resumeFirstBookie();
            AssertExtensions.assertThrows("Bookies should be running, but they aren't", this::restartFirstBookie, ex -> ex instanceof IllegalStateException);
        }
        // Wait for all writes to complete, then reassemble the data in the order set by LogAddress.
        val addresses = Futures.allOfWithResults(futures).join();
        for (int i = 0; i < dataList.size(); i++) {
            writeData.put(addresses.get(i), dataList.get(i));
        }
    }
    // Verify data.
    try (DurableDataLog log = createDurableDataLog()) {
        log.initialize(TIMEOUT);
        verifyReads(log, writeData);
    }
}

Example 8 with CompositeByteArraySegment

use of io.pravega.common.util.CompositeByteArraySegment in project pravega by pravega.

the class BookKeeperLogTests method testReadWithBadLogId.

/**
 * Tests the ability to reject reading from a Log if it has been found it contains Ledgers that do not belong to it.
 */
@Test
public void testReadWithBadLogId() throws Exception {
    final BookKeeper bk = this.factory.get().getBookKeeperClient();
    try (val log = (BookKeeperLog) createDurableDataLog()) {
        log.initialize(TIMEOUT);
        log.append(new CompositeByteArraySegment(new byte[100]), TIMEOUT).join();
    }
    // Add some bad ledgers to the log's metadata.
    val writeLog = (BookKeeperLog) createDurableDataLog();
    val wrapper = this.factory.get().createDebugLogWrapper(writeLog.getLogId());
    wrapper.disable();
    val ledgerNoLogId = createCustomLedger(null);
    val corruptedId = new HashMap<>(Ledgers.createLedgerCustomMetadata(CONTAINER_ID));
    corruptedId.put(Ledgers.PROPERTY_LOG_ID, "abc".getBytes());
    val ledgerBadLogId = createCustomLedger(corruptedId);
    val ledgerOtherLogId = createCustomLedger(Ledgers.createLedgerCustomMetadata(CONTAINER_ID + 1));
    val ledgerGoodLogId = Ledgers.create(bk, this.config.get(), CONTAINER_ID);
    val candidateLedgers = Arrays.asList(ledgerNoLogId, ledgerBadLogId, ledgerOtherLogId, ledgerGoodLogId);
    for (val lh : candidateLedgers) {
        lh.append(new byte[100]);
    }
    // Persist the metadata with bad ledgers.
    val initialMetadata = wrapper.fetchMetadata();
    val newLedgers = new ArrayList<LedgerMetadata>();
    newLedgers.addAll(initialMetadata.getLedgers());
    candidateLedgers.forEach(l -> newLedgers.add(new LedgerMetadata(l.getId(), newLedgers.size() + 1)));
    val badMetadata = LogMetadata.builder().enabled(false).epoch(initialMetadata.getEpoch() + 1).truncationAddress(LogMetadata.INITIAL_TRUNCATION_ADDRESS).updateVersion(wrapper.fetchMetadata().getUpdateVersion()).ledgers(newLedgers).build();
    writeLog.overWriteMetadata(badMetadata);
    // Perform an initial read. This should fail.
    val readLog = (BookKeeperLog) createDurableDataLog();
    readLog.enable();
    readLog.initialize(TIMEOUT);
    @Cleanup val reader = readLog.getReader();
    AssertExtensions.assertThrows("No read exception thrown.", () -> {
        while (reader.getNext() != null) {
        // This is intentionally left blank. We do not care what we read back.
        }
    }, ex -> ex instanceof DataLogCorruptedException);
    // Perform reconciliation.
    BookKeeperLog reconcileLog = (BookKeeperLog) createDurableDataLog();
    DebugBookKeeperLogWrapper reconcileWrapper = this.factory.get().createDebugLogWrapper(reconcileLog.getLogId());
    reconcileWrapper.disable();
    val allLedgers = new ArrayList<ReadHandle>();
    for (LedgerMetadata lm : reconcileWrapper.fetchMetadata().getLedgers()) {
        allLedgers.add(reconcileWrapper.openLedgerNoFencing(lm));
    }
    boolean isChanged = reconcileWrapper.reconcileLedgers(allLedgers);
    Assert.assertTrue("Expected something to change.", isChanged);
    // There should only be two ledgers that survived: the one where we wrote to the original log and ledgerGoodLogId.
    checkLogReadAfterReconciliation(2);
}

Example 9 with CompositeByteArraySegment

use of io.pravega.common.util.CompositeByteArraySegment in project pravega by pravega.

the class BookKeeperLogTests method testReconcileLedgers.

/**
 * Tests {@link DebugBookKeeperLogWrapper#reconcileLedgers}.
 */
@Test
public void testReconcileLedgers() throws Exception {
    final int initialLedgerCount = 5;
    final int midPoint = initialLedgerCount / 2;
    final BookKeeper bk = this.factory.get().getBookKeeperClient();
    // BK Ledger that belongs to this log, but has small id.
    long sameLogSmallIdLedger = -1;
    // Create a Log and add a few ledgers.
    for (int i = 0; i < initialLedgerCount; i++) {
        try (BookKeeperLog log = (BookKeeperLog) createDurableDataLog()) {
            log.initialize(TIMEOUT);
            log.append(new CompositeByteArraySegment(getWriteData()), TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        }
        if (i == midPoint) {
            // We need to create this ledger now. We have no control over the assigned Ledger Id, but we are guaranteed
            // for them to be generated monotonically increasing.
            sameLogSmallIdLedger = createAndCloseLedger(CONTAINER_ID, false);
        }
    }
    // Verify we cannot do this while the log is enabled. This also helps us with getting the final list of ledgers
    // before reconciliation.
    BookKeeperLog log = (BookKeeperLog) createDurableDataLog();
    val wrapper = this.factory.get().createDebugLogWrapper(log.getLogId());
    AssertExtensions.assertThrows("reconcileLedgers worked with non-disabled log.", () -> wrapper.reconcileLedgers(Collections.emptyList()), ex -> ex instanceof IllegalStateException);
    wrapper.disable();
    val initialMetadata = wrapper.fetchMetadata();
    val expectedLedgers = new ArrayList<>(initialMetadata.getLedgers());
    // The last ledger has been created as part of our last log creation. It will be empty and thus will be removed.
    expectedLedgers.remove(expectedLedgers.size() - 1);
    // Simulate the deletion of one of those ledgers.
    final LedgerMetadata deletedLedger = expectedLedgers.get(midPoint);
    expectedLedgers.remove(deletedLedger);
    // Add remaining (valid) ledgers to candidate list.
    val candidateLedgers = new ArrayList<ReadHandle>();
    for (val lm : expectedLedgers) {
        candidateLedgers.add(Ledgers.openFence(lm.getLedgerId(), bk, this.config.get()));
    }
    candidateLedgers.add(Ledgers.openFence(sameLogSmallIdLedger, bk, this.config.get()));
    // Create a ledger that has a high id, but belongs to a different log.
    long differentLogLedger = createAndCloseLedger(CONTAINER_ID + 1, false);
    candidateLedgers.add(Ledgers.openFence(differentLogLedger, bk, this.config.get()));
    // Create a BK ledger that belongs to this log, with high id, but empty.
    long sameLogHighIdEmptyLedger = createAndCloseLedger(CONTAINER_ID, true);
    candidateLedgers.add(Ledgers.openFence(sameLogHighIdEmptyLedger, bk, this.config.get()));
    // Create a BK ledger that belongs to this log, with high id, and not empty. This is the only one expected to be
    // added.
    long sameLogHighIdNonEmptyLedger = createAndCloseLedger(CONTAINER_ID, false);
    expectedLedgers.add(new LedgerMetadata(sameLogHighIdNonEmptyLedger, expectedLedgers.get(expectedLedgers.size() - 1).getSequence() + 1));
    candidateLedgers.add(Ledgers.openFence(sameLogHighIdNonEmptyLedger, bk, this.config.get()));
    // Perform reconciliation.
    boolean isChanged = wrapper.reconcileLedgers(candidateLedgers);
    Assert.assertTrue("Expected first reconcileLedgers to have changed something.", isChanged);
    isChanged = wrapper.reconcileLedgers(candidateLedgers);
    Assert.assertFalse("No expecting second reconcileLedgers to have changed anything.", isChanged);
    // Validate new metadata.
    val newMetadata = wrapper.fetchMetadata();
    Assert.assertFalse("Expected metadata to still be disabled..", newMetadata.isEnabled());
    Assert.assertEquals("Expected epoch to increase.", initialMetadata.getEpoch() + 1, newMetadata.getEpoch());
    Assert.assertEquals("Expected update version to increase.", initialMetadata.getUpdateVersion() + 1, newMetadata.getUpdateVersion());
    Assert.assertEquals("Not expected truncation address to change.", initialMetadata.getTruncationAddress(), newMetadata.getTruncationAddress());
    AssertExtensions.assertListEquals("Unexpected ledger list.", expectedLedgers, newMetadata.getLedgers(), (e, a) -> e.getLedgerId() == a.getLedgerId() && e.getSequence() == a.getSequence());
    checkLogReadAfterReconciliation(expectedLedgers.size());
}

Example 10 with CompositeByteArraySegment

use of io.pravega.common.util.CompositeByteArraySegment in project pravega by pravega.

the class WriteQueueTests method testGetWritesToExecute.

/**
 * Tests the getWritesToExecute() method.
 */
@Test
public void testGetWritesToExecute() {
    final int ledgerChangeIndex = ITEM_COUNT - 5;
    val q = new WriteQueue();
    val writes = new ArrayList<Write>();
    int ledgerId = 0;
    for (int i = 0; i < ITEM_COUNT; i++) {
        if (i == ledgerChangeIndex) {
            ledgerId++;
        }
        val w = new Write(new CompositeByteArraySegment(i), new TestWriteLedger(ledgerId), new CompletableFuture<>());
        q.add(w);
        writes.add(w);
    }
    // 1. Max size reached.
    int sizeLimit = 10;
    val maxSizeResult = q.getWritesToExecute(sizeLimit);
    val expectedMaxSizeResult = new ArrayList<Write>();
    for (Write w : writes) {
        if (w.getLength() > sizeLimit) {
            break;
        }
        sizeLimit -= w.getLength();
        expectedMaxSizeResult.add(w);
    }
    AssertExtensions.assertListEquals("Unexpected writes fetched with size limit.", expectedMaxSizeResult, maxSizeResult, Object::equals);
    // 2. Complete a few writes, then mark a few as in progress.
    writes.get(0).setEntryId(0);
    writes.get(0).complete();
    writes.get(1).beginAttempt();
    val result1 = q.getWritesToExecute(Long.MAX_VALUE);
    // We expect to skip over the first one and second one, but count the second one when doing throttling.
    AssertExtensions.assertListEquals("Unexpected writes fetched when some writes in progress (at beginning).", writes.subList(2, ledgerChangeIndex), result1, Object::equals);
    // 3. Mark a few writes as in progress after a non-progress write.
    writes.get(3).beginAttempt();
    val result2 = q.getWritesToExecute(Long.MAX_VALUE);
    Assert.assertEquals("Unexpected writes fetched when in-progress writes exist after non-in-progress writes.", 0, result2.size());
    // 4. LedgerChange.
    int beginIndex = ledgerChangeIndex - 5;
    for (int i = 0; i < beginIndex; i++) {
        writes.get(i).setEntryId(i);
        writes.get(i).complete();
    }
    q.removeFinishedWrites();
    val result3 = q.getWritesToExecute(Long.MAX_VALUE);
    AssertExtensions.assertListEquals("Unexpected writes fetched when ledger changed.", writes.subList(beginIndex, ledgerChangeIndex), result3, Object::equals);
    result3.forEach(w -> w.setEntryId(0));
    result3.forEach(Write::complete);
    q.removeFinishedWrites();
    val result4 = q.getWritesToExecute(Long.MAX_VALUE);
    AssertExtensions.assertListEquals("Unexpected writes fetched from the end, after ledger changed.", writes.subList(ledgerChangeIndex, writes.size()), result4, Object::equals);
}