Examples with TableEntry io.pravega.segmentstore.contracts.tables.TableEntry used on opensource projects

Example 11 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class EntrySerializerTests method generateEntries.

private List<TableEntry> generateEntries() {
    val rnd = new Random(0);
    AtomicBoolean generatedEmpty = new AtomicBoolean(false);
    return generateKeys(rnd).stream().map(key -> {
        byte[] value = new byte[generatedEmpty.get() ? rnd.nextInt(MAX_VALUE_SIZE) : 0];
        generatedEmpty.set(true);
        rnd.nextBytes(value);
        return TableEntry.versioned(key.getKey(), new ByteArraySegment(value), Math.max(0, rnd.nextLong()));
    }).collect(Collectors.toList());
}

Example 12 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class HashTableSegmentLayoutTests method testRecovery.

/**
 * Tests the ability to resume operations after a recovery event. Scenarios include:
 * - Index is up-to-date ({@link TableAttributes#INDEX_OFFSET} equals Segment.Length.
 * - Index is not up-to-date ({@link TableAttributes#INDEX_OFFSET} is less than Segment.Length.
 */
@Test
public void testRecovery() throws Exception {
    // Generate a set of TestEntryData (List<TableEntry>, ExpectedResults.
    // Process each TestEntryData in turn.  After each time, re-create the Extension.
    // Verify gets are blocked on indexing. Then index, verify unblocked and then re-create the Extension, and verify again.
    val recoveryConfig = TableExtensionConfig.builder().with(TableExtensionConfig.MAX_TAIL_CACHE_PREINDEX_BATCH_SIZE, (MAX_KEY_LENGTH + MAX_VALUE_LENGTH) * 11).build();
    @Cleanup val context = new TableContext(recoveryConfig, executorService());
    // Create the Segment.
    context.ext.createSegment(SEGMENT_NAME, SegmentType.TABLE_SEGMENT_HASH, TIMEOUT).join();
    // Close the initial extension, as we don't need it anymore.
    context.ext.close();
    // Generate test data (in update & remove batches).
    val data = generateTestData(context);
    // Process each such batch in turn.
    for (int i = 0; i < data.size(); i++) {
        val current = data.get(i);
        // of this is writing the data to the Segment.
        try (val ext = context.createExtension()) {
            val toUpdate = current.toUpdate.entrySet().stream().map(e -> toUnconditionalTableEntry(e.getKey(), e.getValue(), 0)).collect(Collectors.toList());
            ext.put(SEGMENT_NAME, toUpdate, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            val toRemove = current.toRemove.stream().map(k -> toUnconditionalKey(k, 0)).collect(Collectors.toList());
            ext.remove(SEGMENT_NAME, toRemove, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
        }
        // Create a new instance of the extension (which simulates a recovery) and verify it exhibits the correct behavior.
        try (val ext = context.createExtension()) {
            // We should have unindexed data.
            long lastIndexedOffset = context.segment().getInfo().getAttributes().get(TableAttributes.INDEX_OFFSET);
            long segmentLength = context.segment().getInfo().getLength();
            AssertExtensions.assertGreaterThan("Expected some unindexed data.", lastIndexedOffset, segmentLength);
            // This ensures that last iteration uses the processor.
            boolean useProcessor = i % 2 == 0;
            // Verify get requests are blocked.
            val key1 = current.expectedEntries.keySet().stream().findFirst().orElse(null);
            val get1 = ext.get(SEGMENT_NAME, Collections.singletonList(key1), TIMEOUT);
            val getResult1 = get1.get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            Assert.assertEquals("Unexpected completion result for recovered get.", current.expectedEntries.get(key1), getResult1.get(0).getValue());
            if (useProcessor) {
                // Create, populate, and flush the processor.
                @Cleanup val processor = (WriterTableProcessor) ext.createWriterSegmentProcessors(context.segment().getMetadata()).stream().findFirst().orElse(null);
                addToProcessor(lastIndexedOffset, (int) (segmentLength - lastIndexedOffset), processor);
                processor.flush(TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
                Assert.assertFalse("Unexpected result from WriterTableProcessor.mustFlush() after flushing.", processor.mustFlush());
            }
        }
    }
    // Verify final result. We create yet another extension here, and purposefully do not instantiate any writer processors;
    // we want to make sure the data are accessible even without that being created (since the indexing is all caught up).
    @Cleanup val ext2 = context.createExtension();
    check(data.get(data.size() - 1).expectedEntries, Collections.emptyList(), ext2);
}

Example 13 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class WriterTableProcessorTests method generateAndPopulateEntries.

private ArrayList<TestBatchData> generateAndPopulateEntries(TestContext context) {
    val result = new ArrayList<TestBatchData>();
    int count = 0;
    while (count < UPDATE_COUNT) {
        int batchSize = Math.min(UPDATE_BATCH_SIZE, UPDATE_COUNT - count);
        Map<BufferView, TableEntry> prevState = result.isEmpty() ? Collections.emptyMap() : result.get(result.size() - 1).expectedEntries;
        result.add(generateAndPopulateEntriesBatch(batchSize, prevState, context));
        count += batchSize;
    }
    return result;
}

Example 14 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class TableEntryDeltaIteratorTests method test.

@SneakyThrows
private void test(TestData testData, TableEntry startEntry) {
    val actual = testData.getEntriesFromIteration(startEntry.getKey());
    // Get a sub-list, starting at the same TableKey in the 'expected' list.
    List<TableEntry> expected = new ArrayList<>();
    for (int i = 0; i < testData.getExpected().size(); i++) {
        if (compareKeys(testData.getExpected().get(i), startEntry)) {
            expected = testData.getExpected().subList(i, testData.getExpected().size());
            break;
        }
    }
    testData.setActualEntries(actual);
    Assert.assertEquals(actual.size(), expected.size());
    AssertExtensions.assertListEquals("Assert equivalency by TableKey and Value (ignoring version).", expected, actual, (t1, t2) -> compareKeys(t1, t2) && t1.getValue().equals(t2.getValue()));
}

Example 15 with TableEntry

use of io.pravega.segmentstore.contracts.tables.TableEntry in project pravega by pravega.

the class SegmentStoreAdapter method updateTableEntry.

@Override
public CompletableFuture<Long> updateTableEntry(String tableName, BufferView key, BufferView value, Long compareVersion, Duration timeout) {
    ensureRunning();
    TableEntry e = compareVersion == null || compareVersion == TableKey.NO_VERSION ? TableEntry.unversioned(key, value) : TableEntry.versioned(key, value, compareVersion);
    return this.tableStore.put(tableName, Collections.singletonList(e), timeout).thenApply(versions -> versions.get(0));
}