Examples with TestDurableDataLog io.pravega.segmentstore.server.TestDurableDataLog used on opensource projects

Example 11 with TestDurableDataLog

use of io.pravega.segmentstore.server.TestDurableDataLog in project pravega by pravega.

the class DataFrameBuilderTests method testAppendWithSerializationFailure.

/**
 * Tests the case when the appends fail because of Serialization failures.
 * Serialization errors should only affect the append that caused it. It should not cause any data to be dropped
 * or put the DataFrameBuilder in a stuck state.
 * This should be done both with large and with small LogItems. Large items span multiple frames.
 */
@Test
public void testAppendWithSerializationFailure() throws Exception {
    // Fail every X records.
    int failEvery = 7;
    ArrayList<TestLogItem> records = DataFrameTestHelpers.generateLogItems(RECORD_COUNT / 2, SMALL_RECORD_MIN_SIZE, SMALL_RECORD_MAX_SIZE, 0);
    records.addAll(DataFrameTestHelpers.generateLogItems(RECORD_COUNT / 2, LARGE_RECORD_MIN_SIZE, LARGE_RECORD_MAX_SIZE, records.size()));
    // Have every other 'failEvery' record fail after writing 90% of itself.
    for (int i = 0; i < records.size(); i += failEvery) {
        records.get(i).failSerializationAfterComplete(0.9, new IOException("intentional " + i));
    }
    HashSet<Integer> failedIndices = new HashSet<>();
    val order = new HashMap<DataFrameBuilder.CommitArgs, Integer>();
    try (TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, FRAME_SIZE, executorService())) {
        dataLog.initialize(TIMEOUT);
        List<DataFrameBuilder.CommitArgs> commitFrames = Collections.synchronizedList(new ArrayList<>());
        BiConsumer<Throwable, DataFrameBuilder.CommitArgs> errorCallback = (ex, a) -> Assert.fail(String.format("Unexpected error occurred upon commit. %s", ex));
        val args = new DataFrameBuilder.Args(DataFrameTestHelpers.appendOrder(order), commitFrames::add, errorCallback, executorService());
        try (DataFrameBuilder<TestLogItem> b = new DataFrameBuilder<>(dataLog, SERIALIZER, args)) {
            for (int i = 0; i < records.size(); i++) {
                try {
                    b.append(records.get(i));
                } catch (IOException ex) {
                    failedIndices.add(i);
                }
            }
        }
        // Read all entries in the Log and interpret them as DataFrames, then verify the records can be reconstructed.
        await(() -> commitFrames.size() >= order.size(), 20);
        List<DataFrame.DataFrameEntryIterator> frames = dataLog.getAllEntries(readItem -> DataFrame.read(readItem.getPayload(), readItem.getLength(), readItem.getAddress()));
        Assert.assertEquals("Unexpected number of frames generated.", commitFrames.size(), frames.size());
        // Check the correctness of the commit callback.
        AssertExtensions.assertGreaterThan("Not enough Data Frames were generated.", 1, commitFrames.size());
        AssertExtensions.assertGreaterThan("Not enough LogItems were failed.", records.size() / failEvery, failedIndices.size());
        DataFrameTestHelpers.checkReadRecords(frames, records, failedIndices, r -> new ByteArraySegment(r.getFullSerialization()));
    }
}

Example 12 with TestDurableDataLog

use of io.pravega.segmentstore.server.TestDurableDataLog in project pravega by pravega.

the class DataFrameBuilderTests method testAppendWithCommitFailure.

/**
 * Tests the case when the DataLog fails to commit random frames.
 * Commit errors should affect only the LogItems that were part of it. It should cause data to be dropped
 * and affected appends failed.
 * This should be done both with large and with small LogItems. Large items span multiple frames.
 */
@Test
public void testAppendWithCommitFailure() throws Exception {
    // Fail the commit to DurableDataLog after this many writes.
    int failAt = 7;
    List<TestLogItem> records = DataFrameTestHelpers.generateLogItems(RECORD_COUNT / 2, SMALL_RECORD_MIN_SIZE, SMALL_RECORD_MAX_SIZE, 0);
    records.addAll(DataFrameTestHelpers.generateLogItems(RECORD_COUNT / 2, LARGE_RECORD_MIN_SIZE, LARGE_RECORD_MAX_SIZE, records.size()));
    @Cleanup TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, FRAME_SIZE, executorService());
    dataLog.initialize(TIMEOUT);
    val asyncInjector = new ErrorInjector<Exception>(count -> count >= failAt, IntentionalException::new);
    dataLog.setAppendErrorInjectors(null, asyncInjector);
    AtomicInteger failCount = new AtomicInteger();
    List<DataFrameBuilder.CommitArgs> successCommits = Collections.synchronizedList(new ArrayList<>());
    // Keep a reference to the builder (once created) so we can inspect its failure cause).
    val builderRef = new AtomicReference<DataFrameBuilder>();
    val attemptCount = new AtomicInteger();
    BiConsumer<Throwable, DataFrameBuilder.CommitArgs> errorCallback = (ex, a) -> {
        attemptCount.decrementAndGet();
        // Check that we actually did want an exception to happen.
        Throwable expectedError = Exceptions.unwrap(asyncInjector.getLastCycleException());
        Assert.assertNotNull("An error happened but none was expected: " + ex, expectedError);
        Throwable actualError = Exceptions.unwrap(ex);
        if (!(ex instanceof ObjectClosedException)) {
            // First failure.
            Assert.assertEquals("Unexpected error occurred upon commit.", expectedError, actualError);
        }
        if (builderRef.get().failureCause() != null) {
            checkFailureCause(builderRef.get(), ce -> ce instanceof IntentionalException);
        }
        failCount.incrementAndGet();
    };
    val args = new DataFrameBuilder.Args(ca -> attemptCount.incrementAndGet(), successCommits::add, errorCallback, executorService());
    try (DataFrameBuilder<TestLogItem> b = new DataFrameBuilder<>(dataLog, SERIALIZER, args)) {
        builderRef.set(b);
        try {
            for (val r : records) {
                b.append(r);
            }
            b.close();
        } catch (ObjectClosedException ex) {
            await(() -> b.failureCause() != null, 20);
            // If DataFrameBuilder is closed, then we must have had an exception thrown via the callback before.
            Assert.assertNotNull("DataFrameBuilder is closed, yet failure cause is not set yet.", b.failureCause());
            checkFailureCause(b, ce -> ce instanceof IntentionalException);
        }
    }
    await(() -> successCommits.size() >= attemptCount.get(), 20);
    // Read all committed items.
    val reader = new DataFrameReader<TestLogItem>(dataLog, new TestSerializer(), CONTAINER_ID);
    val readItems = new ArrayList<TestLogItem>();
    DataFrameRecord<TestLogItem> readItem;
    while ((readItem = reader.getNext()) != null) {
        readItems.add(readItem.getItem());
    }
    val lastCommitSeqNo = successCommits.stream().mapToLong(DataFrameBuilder.CommitArgs::getLastFullySerializedSequenceNumber).max().orElse(-1);
    val expectedItems = records.stream().filter(r -> r.getSequenceNumber() <= lastCommitSeqNo).collect(Collectors.toList());
    AssertExtensions.assertListEquals("Items read back do not match expected values.", expectedItems, readItems, TestLogItem::equals);
    // Read all entries in the Log and interpret them as DataFrames, then verify the records can be reconstructed.
    val frames = dataLog.getAllEntries(ri -> DataFrame.read(ri.getPayload(), ri.getLength(), ri.getAddress()));
    // Check the correctness of the commit callback.
    AssertExtensions.assertGreaterThan("Not enough Data Frames were generated.", 1, frames.size());
    Assert.assertEquals("Unexpected number of frames generated.", successCommits.size(), frames.size());
}

Example 13 with TestDurableDataLog

use of io.pravega.segmentstore.server.TestDurableDataLog in project pravega by pravega.

the class DataFrameBuilderTests method testFlush.

/**
 * Tests the flush() method.
 */
@Test
public void testFlush() throws Exception {
    // Append two records, make sure they are not flushed, close the Builder, then make sure they are flushed.
    try (TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, FRAME_SIZE, executorService())) {
        dataLog.initialize(TIMEOUT);
        ArrayList<TestLogItem> records = DataFrameTestHelpers.generateLogItems(2, SMALL_RECORD_MIN_SIZE, SMALL_RECORD_MAX_SIZE, 0);
        List<DataFrameBuilder.CommitArgs> commitFrames = Collections.synchronizedList(new ArrayList<>());
        BiConsumer<Throwable, DataFrameBuilder.CommitArgs> errorCallback = (ex, a) -> Assert.fail(String.format("Unexpected error occurred upon commit. %s", ex));
        val args = new DataFrameBuilder.Args(Callbacks::doNothing, commitFrames::add, errorCallback, executorService());
        @Cleanup DataFrameBuilder<TestLogItem> b = new DataFrameBuilder<>(dataLog, SERIALIZER, args);
        for (TestLogItem item : records) {
            b.append(item);
        }
        // Check the correctness of the commit callback.
        Assert.assertEquals("A Data Frame was generated but none was expected yet.", 0, commitFrames.size());
        // Invoke flush.
        b.flush();
        // Wait for all the frames commit callbacks to be invoked.
        await(() -> commitFrames.size() >= 1, 20);
        // Check the correctness of the commit callback (after closing the builder).
        Assert.assertEquals("Exactly one Data Frame was expected so far.", 1, commitFrames.size());
        // Read all entries in the Log and interpret them as DataFrames, then verify the records can be reconstructed.
        val frames = dataLog.getAllEntries(readItem -> DataFrame.read(readItem.getPayload(), readItem.getLength(), readItem.getAddress()));
        Assert.assertEquals("Unexpected number of frames generated.", commitFrames.size(), frames.size());
        DataFrameTestHelpers.checkReadRecords(frames, records, r -> new ByteArraySegment(r.getFullSerialization()));
    }
}

Example 14 with TestDurableDataLog

use of io.pravega.segmentstore.server.TestDurableDataLog in project pravega by pravega.

the class DataFrameReaderTests method testReadsWithPartialEntries.

/**
 * Tests the case when we begin reading from a DataFrame which begins with a partial record. That record needs to
 * be dropped (not returned). DataFrameReader should always return full records.
 */
@Test
public void testReadsWithPartialEntries() throws Exception {
    // This test will only work if LARGE_RECORD_MIN_SIZE > FRAME_SIZE.
    ArrayList<TestLogItem> records = DataFrameTestHelpers.generateLogItems(3, LARGE_RECORD_MIN_SIZE, LARGE_RECORD_MIN_SIZE, 0);
    try (TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, FRAME_SIZE, executorService())) {
        dataLog.initialize(TIMEOUT);
        ArrayList<DataFrameBuilder.CommitArgs> commitFrames = new ArrayList<>();
        BiConsumer<Throwable, DataFrameBuilder.CommitArgs> errorCallback = (ex, a) -> Assert.fail(String.format("Unexpected error occurred upon commit. %s", ex));
        val args = new DataFrameBuilder.Args(Callbacks::doNothing, commitFrames::add, errorCallback, executorService());
        try (DataFrameBuilder<TestLogItem> b = new DataFrameBuilder<>(dataLog, SERIALIZER, args)) {
            for (TestLogItem r : records) {
                b.append(r);
            }
            b.flush();
        }
        // Delete the first entry in the DataLog.
        ArrayList<Integer> failedIndices = new ArrayList<>();
        dataLog.truncate(commitFrames.get(0).getLogAddress(), TIMEOUT).join();
        // Given that each TestLogItem's length is larger than a data frame, truncating the first DataFrame will
        // invalidate the first one.
        failedIndices.add(0);
        TestSerializer logItemFactory = new TestSerializer();
        DataFrameReader<TestLogItem> reader = new DataFrameReader<>(dataLog, logItemFactory, CONTAINER_ID);
        List<TestLogItem> readItems = readAll(reader);
        checkReadResult(records, failedIndices, readItems);
    }
}

Example 15 with TestDurableDataLog

use of io.pravega.segmentstore.server.TestDurableDataLog in project pravega by pravega.

the class DataFrameReaderTests method testReadsNoFailure.

/**
 * Tests the happy case: DataFrameReader can read from a DataLog when the are no exceptions.
 */
@Test
public void testReadsNoFailure() throws Exception {
    // Fail every X records (write-wise).
    int failEvery = 7;
    ArrayList<TestLogItem> records = DataFrameTestHelpers.generateLogItems(100, SMALL_RECORD_MIN_SIZE, SMALL_RECORD_MAX_SIZE, 0);
    records.addAll(DataFrameTestHelpers.generateLogItems(100, LARGE_RECORD_MIN_SIZE, LARGE_RECORD_MAX_SIZE, records.size()));
    // Have every other 'failEvery' record fail after writing 90% of itself.
    for (int i = 0; i < records.size(); i += failEvery) {
        records.get(i).failSerializationAfterComplete(0.9, new IOException("intentional " + i));
    }
    HashSet<Integer> failedIndices = new HashSet<>();
    try (TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, FRAME_SIZE, executorService())) {
        dataLog.initialize(TIMEOUT);
        BiConsumer<Throwable, DataFrameBuilder.CommitArgs> errorCallback = (ex, a) -> Assert.fail(String.format("Unexpected error occurred upon commit. %s", ex));
        val args = new DataFrameBuilder.Args(Callbacks::doNothing, Callbacks::doNothing, errorCallback, executorService());
        try (DataFrameBuilder<TestLogItem> b = new DataFrameBuilder<>(dataLog, SERIALIZER, args)) {
            for (int i = 0; i < records.size(); i++) {
                try {
                    b.append(records.get(i));
                } catch (IOException ex) {
                    failedIndices.add(i);
                }
            }
            b.flush();
        }
        TestSerializer logItemFactory = new TestSerializer();
        DataFrameReader<TestLogItem> reader = new DataFrameReader<>(dataLog, logItemFactory, CONTAINER_ID);
        List<TestLogItem> readItems = readAll(reader);
        checkReadResult(records, failedIndices, readItems);
    }
}