Examples with LogSegmentMetadata org.apache.distributedlog.LogSegmentMetadata used on opensource projects

Example 31 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class TestBKLogSegmentEntryReader method testReadEntriesFromCompleteLogSegment.

@Test(timeout = 60000)
public void testReadEntriesFromCompleteLogSegment() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(10);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one", 1, segments.size());
    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();
    boolean done = false;
    long txId = 1L;
    long entryId = 0L;
    while (!done) {
        Entry.Reader entryReader;
        try {
            entryReader = Utils.ioResult(reader.readNext(1)).get(0);
        } catch (EndOfLogSegmentException eol) {
            done = true;
            continue;
        }
        LogRecordWithDLSN record = entryReader.nextRecord();
        while (null != record) {
            if (!record.isControl()) {
                DLMTestUtil.verifyLogRecord(record);
                assertEquals(txId, record.getTransactionId());
                ++txId;
            }
            DLSN dlsn = record.getDlsn();
            assertEquals(1L, dlsn.getLogSegmentSequenceNo());
            assertEquals(entryId, dlsn.getEntryId());
            record = entryReader.nextRecord();
        }
        ++entryId;
    }
    assertEquals(21, txId);
    assertFalse(reader.hasCaughtUpOnInprogress());
    Utils.close(reader);
}

Example 32 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class TestZKLogStreamMetadataStore method testGetLogSegments.

@Test(timeout = 60000)
public void testGetLogSegments() throws Exception {
    String logName = testName.getMethodName();
    String logIdentifier = "<default>";
    // create log
    createLog(zkc, uri, logName, logIdentifier, 5);
    List<LogSegmentMetadata> segments = FutureUtils.result(getLogSegments(zkc, LogMetadata.getLogSegmentsPath(uri, logName, logIdentifier)));
    assertEquals(5, segments.size());
    for (int i = 0; i < 5; i++) {
        assertEquals(1L + i, segments.get(i).getLogSegmentSequenceNumber());
    }
}

Example 33 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class TestPerStreamLogSegmentCache method testBasicOperations.

@Test(timeout = 60000)
public void testBasicOperations() {
    LogSegmentMetadata metadata = DLMTestUtil.completedLogSegment("/segment1", 1L, 1L, 100L, 100, 1L, 99L, 0L);
    String name = DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(1L);
    PerStreamLogSegmentCache cache = new PerStreamLogSegmentCache("test-basic-operations");
    assertNull("No log segment " + name + " should be cached", cache.get(name));
    cache.add(name, metadata);
    LogSegmentMetadata metadataRetrieved = cache.get(name);
    assertNotNull("log segment " + name + " should be cached", metadataRetrieved);
    assertEquals("Wrong log segment metadata returned for " + name, metadata, metadataRetrieved);
    LogSegmentMetadata metadataRemoved = cache.remove(name);
    assertNull("log segment " + name + " should be removed from cache", cache.get(name));
    assertEquals("Wrong log segment metadata removed for " + name, metadata, metadataRemoved);
    assertNull("No log segment " + name + " to be removed", cache.remove(name));
}

Example 34 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class TestPerStreamLogSegmentCache method testUpdate.

@Test(timeout = 60000)
public void testUpdate() {
    PerStreamLogSegmentCache cache = new PerStreamLogSegmentCache("test-update");
    // add 5 completed log segments
    for (int i = 1; i <= 5; i++) {
        LogSegmentMetadata metadata = DLMTestUtil.completedLogSegment("/segment" + i, i, i, i * 100L, 100, i, 99L, 0L);
        String name = DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i);
        cache.add(name, metadata);
    }
    // add one inprogress log segment
    LogSegmentMetadata inprogress = DLMTestUtil.inprogressLogSegment("/inprogress-6", 6, 600L, 6);
    String name = DLMTestUtil.inprogressZNodeName(6);
    cache.add(name, inprogress);
    // deleted first 2 completed log segments and completed the last one
    Set<String> segmentRemoved = Sets.newHashSet();
    for (int i = 1; i <= 2; i++) {
        segmentRemoved.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i));
    }
    segmentRemoved.add((DLMTestUtil.inprogressZNodeName(6)));
    Set<String> segmentReceived = Sets.newHashSet();
    Map<String, LogSegmentMetadata> segmentAdded = Maps.newHashMap();
    for (int i = 3; i <= 6; i++) {
        segmentReceived.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i));
        if (i == 6) {
            segmentAdded.put(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(i), DLMTestUtil.completedLogSegment("/segment" + i, i, i, i * 100L, 100, i, 99L, 0L));
        }
    }
    // update the cache
    cache.update(segmentRemoved, segmentAdded);
    for (String segment : segmentRemoved) {
        assertNull("Segment " + segment + " should be removed.", cache.get(segment));
    }
    for (String segment : segmentReceived) {
        assertNotNull("Segment " + segment + " should not be removed", cache.get(segment));
    }
    for (Map.Entry<String, LogSegmentMetadata> entry : segmentAdded.entrySet()) {
        assertEquals("Segment " + entry.getKey() + " should be added.", entry.getValue(), entry.getValue());
    }
}

Example 35 with LogSegmentMetadata

use of org.apache.distributedlog.LogSegmentMetadata in project bookkeeper by apache.

the class TestPerStreamLogSegmentCache method testSameLogSegment.

@Test(timeout = 60000, expected = UnexpectedException.class)
public void testSameLogSegment() throws Exception {
    PerStreamLogSegmentCache cache = new PerStreamLogSegmentCache("test-same-log-segment");
    List<LogSegmentMetadata> expectedList = Lists.newArrayListWithExpectedSize(2);
    LogSegmentMetadata inprogress = DLMTestUtil.inprogressLogSegment("/inprogress-1", 1L, 1L, 1L);
    expectedList.add(inprogress);
    cache.add(DLMTestUtil.inprogressZNodeName(1L), inprogress);
    LogSegmentMetadata completed = DLMTestUtil.completedLogSegment("/segment-1", 1L, 1L, 100L, 100, 1L, 99L, 0L);
    expectedList.add(completed);
    cache.add(DLMTestUtil.completedLedgerZNodeNameWithLogSegmentSequenceNumber(1L), completed);
    cache.getLogSegments(LogSegmentMetadata.COMPARATOR);
}