Examples with DLSN org.apache.distributedlog.DLSN used on opensource projects

Example 1 with DLSN

use of org.apache.distributedlog.DLSN in project bookkeeper by apache.

the class LogSegmentMetadataStoreUpdater method updateLastRecord.

@Override
public CompletableFuture<LogSegmentMetadata> updateLastRecord(LogSegmentMetadata segment, LogRecordWithDLSN record) {
    DLSN dlsn = record.getDlsn();
    checkState(!segment.isInProgress(), "Updating last dlsn for an inprogress log segment isn't supported.");
    checkArgument(segment.isDLSNinThisSegment(dlsn), "DLSN " + dlsn + " doesn't belong to segment " + segment);
    final LogSegmentMetadata newSegment = segment.mutator().setLastDLSN(dlsn).setLastTxId(record.getTransactionId()).setRecordCount(record).build();
    return updateSegmentMetadata(newSegment);
}

Example 2 with DLSN

use of org.apache.distributedlog.DLSN in project bookkeeper by apache.

the class TestBKLogSegmentEntryReader method testMaxPrefetchEntriesSmallBatch.

@Test(timeout = 60000)
public void testMaxPrefetchEntriesSmallBatch() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(2);
    confLocal.setMaxPrefetchEntriesPerLogSegment(10);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one", 1, segments.size());
    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();
    // wait for the read ahead entries to become available
    while (reader.readAheadEntries.size() < 10) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    long txId = 1L;
    long entryId = 0L;
    assertEquals(10, reader.readAheadEntries.size());
    assertEquals(10, reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());
    // read first entry
    Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    while (null != record) {
        if (!record.isControl()) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(txId, record.getTransactionId());
            ++txId;
        }
        DLSN dlsn = record.getDlsn();
        assertEquals(1L, dlsn.getLogSegmentSequenceNo());
        assertEquals(entryId, dlsn.getEntryId());
        record = entryReader.nextRecord();
    }
    ++entryId;
    assertEquals(2L, txId);
    // wait for the read ahead entries to become 10 again
    while (reader.readAheadEntries.size() < 10) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals(10, reader.readAheadEntries.size());
    assertEquals(11, reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());
    Utils.close(reader);
}

Example 3 with DLSN

use of org.apache.distributedlog.DLSN in project bookkeeper by apache.

the class TestBKLogSegmentEntryReader method testReadEntriesFromInprogressSegment.

@Test(timeout = 60000)
public void testReadEntriesFromInprogressSegment() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(20);
    confLocal.setMaxPrefetchEntriesPerLogSegment(20);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    AsyncLogWriter writer = createInprogressLogSegment(dlm, confLocal, 5);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one", 1, segments.size());
    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();
    long expectedLastAddConfirmed = 8L;
    // wait until sending out all prefetch requests
    while (reader.readAheadEntries.size() < expectedLastAddConfirmed + 2) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals(expectedLastAddConfirmed + 2, reader.getNextEntryId());
    long txId = 1L;
    long entryId = 0L;
    while (true) {
        Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
        LogRecordWithDLSN record = entryReader.nextRecord();
        while (null != record) {
            if (!record.isControl()) {
                DLMTestUtil.verifyLogRecord(record);
                assertEquals(txId, record.getTransactionId());
                ++txId;
            }
            DLSN dlsn = record.getDlsn();
            assertEquals(1L, dlsn.getLogSegmentSequenceNo());
            assertEquals(entryId, dlsn.getEntryId());
            record = entryReader.nextRecord();
        }
        ++entryId;
        if (entryId == expectedLastAddConfirmed + 1) {
            break;
        }
    }
    assertEquals(6L, txId);
    CompletableFuture<List<Entry.Reader>> nextReadFuture = reader.readNext(1);
    // write another record to commit previous writes
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId)));
    // the long poll will be satisfied
    List<Entry.Reader> nextReadEntries = Utils.ioResult(nextReadFuture);
    assertEquals(1, nextReadEntries.size());
    assertTrue(reader.hasCaughtUpOnInprogress());
    Entry.Reader entryReader = nextReadEntries.get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    assertNotNull(record);
    assertTrue(record.isControl());
    assertNull(entryReader.nextRecord());
    // once the read is advanced, we will prefetch next record
    while (reader.getNextEntryId() <= entryId) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals(entryId + 2, reader.getNextEntryId());
    assertEquals(1, reader.readAheadEntries.size());
    Utils.close(reader);
    Utils.close(writer);
}

Example 4 with DLSN

use of org.apache.distributedlog.DLSN in project bookkeeper by apache.

the class TestBKLogSegmentEntryReader method testMaxPrefetchEntriesLargeBatch.

@Test(timeout = 60000)
public void testMaxPrefetchEntriesLargeBatch() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(5);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one", 1, segments.size());
    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();
    // wait for the read ahead entries to become available
    while (reader.readAheadEntries.size() < 5) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    long txId = 1L;
    long entryId = 0L;
    assertEquals(5, reader.readAheadEntries.size());
    assertEquals(5, reader.getNextEntryId());
    // read first entry
    Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    while (null != record) {
        if (!record.isControl()) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(txId, record.getTransactionId());
            ++txId;
        }
        DLSN dlsn = record.getDlsn();
        assertEquals(1L, dlsn.getLogSegmentSequenceNo());
        assertEquals(entryId, dlsn.getEntryId());
        record = entryReader.nextRecord();
    }
    ++entryId;
    assertEquals(2L, txId);
    // wait for the read ahead entries to become 10 again
    while (reader.readAheadEntries.size() < 5) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals(5, reader.readAheadEntries.size());
    assertEquals(6, reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());
    Utils.close(reader);
}

Example 5 with DLSN

use of org.apache.distributedlog.DLSN in project bookkeeper by apache.

the class TestLogSegmentMetadataStoreUpdater method testUpdateLastDLSN.

@Test(timeout = 60000)
public void testUpdateLastDLSN() throws Exception {
    String ledgerPath = "/testUpdateLastDLSN";
    zkc.get().create(ledgerPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    // Create 1 completed log segment
    LogSegmentMetadata completedLogSegment = DLMTestUtil.completedLogSegment(ledgerPath, 1L, 0L, 99L, 100, 1L, 99L, 0L);
    completedLogSegment.write(zkc);
    // Create 1 inprogress log segment
    LogSegmentMetadata inprogressLogSegment = DLMTestUtil.inprogressLogSegment(ledgerPath, 2L, 100L, 2L);
    inprogressLogSegment.write(zkc);
    DLSN badLastDLSN = new DLSN(99L, 0L, 0L);
    DLSN goodLastDLSN1 = new DLSN(1L, 100L, 0L);
    DLSN goodLastDLSN2 = new DLSN(2L, 200L, 0L);
    LogRecordWithDLSN badRecord = DLMTestUtil.getLogRecordWithDLSNInstance(badLastDLSN, 100L);
    LogRecordWithDLSN goodRecord1 = DLMTestUtil.getLogRecordWithDLSNInstance(goodLastDLSN1, 100L);
    LogRecordWithDLSN goodRecord2 = DLMTestUtil.getLogRecordWithDLSNInstance(goodLastDLSN2, 200L);
    // Dryrun
    MetadataUpdater dryrunUpdater = new DryrunLogSegmentMetadataStoreUpdater(conf, metadataStore);
    try {
        Utils.ioResult(dryrunUpdater.updateLastRecord(completedLogSegment, badRecord));
        fail("Should fail on updating dlsn that in different log segment");
    } catch (IllegalArgumentException iae) {
    // expected
    }
    try {
        Utils.ioResult(dryrunUpdater.updateLastRecord(inprogressLogSegment, goodRecord2));
        fail("Should fail on updating dlsn for an inprogress log segment");
    } catch (IllegalStateException ise) {
    // expected
    }
    LogSegmentMetadata updatedCompletedLogSegment = Utils.ioResult(dryrunUpdater.updateLastRecord(completedLogSegment, goodRecord1));
    assertEquals(goodLastDLSN1, updatedCompletedLogSegment.getLastDLSN());
    assertEquals(goodRecord1.getTransactionId(), updatedCompletedLogSegment.getLastTxId());
    assertTrue(updatedCompletedLogSegment.isRecordLastPositioninThisSegment(goodRecord1));
    Map<Long, LogSegmentMetadata> segmentList = readLogSegments(ledgerPath);
    assertEquals(2, segmentList.size());
    LogSegmentMetadata readCompletedLogSegment = segmentList.get(1L);
    assertNotNull(readCompletedLogSegment);
    assertEquals(completedLogSegment, readCompletedLogSegment);
    LogSegmentMetadata readInprogressLogSegment = segmentList.get(2L);
    assertNotNull(readInprogressLogSegment);
    assertEquals(inprogressLogSegment, readInprogressLogSegment);
    // Fix the last dlsn
    MetadataUpdater updater = LogSegmentMetadataStoreUpdater.createMetadataUpdater(conf, metadataStore);
    try {
        Utils.ioResult(updater.updateLastRecord(completedLogSegment, badRecord));
        fail("Should fail on updating dlsn that in different log segment");
    } catch (IllegalArgumentException iae) {
    // expected
    }
    try {
        Utils.ioResult(updater.updateLastRecord(inprogressLogSegment, goodRecord2));
        fail("Should fail on updating dlsn for an inprogress log segment");
    } catch (IllegalStateException ise) {
    // expected
    }
    updatedCompletedLogSegment = Utils.ioResult(updater.updateLastRecord(completedLogSegment, goodRecord1));
    assertEquals(goodLastDLSN1, updatedCompletedLogSegment.getLastDLSN());
    assertEquals(goodRecord1.getTransactionId(), updatedCompletedLogSegment.getLastTxId());
    assertTrue(updatedCompletedLogSegment.isRecordLastPositioninThisSegment(goodRecord1));
    segmentList = readLogSegments(ledgerPath);
    assertEquals(2, segmentList.size());
    readCompletedLogSegment = segmentList.get(1L);
    assertNotNull(readCompletedLogSegment);
    assertEquals(goodLastDLSN1, readCompletedLogSegment.getLastDLSN());
    assertEquals(goodRecord1.getTransactionId(), readCompletedLogSegment.getLastTxId());
    assertTrue(readCompletedLogSegment.isRecordLastPositioninThisSegment(goodRecord1));
    assertEquals(updatedCompletedLogSegment, readCompletedLogSegment);
    assertEquals(completedLogSegment.getCompletionTime(), readCompletedLogSegment.getCompletionTime());
    assertEquals(completedLogSegment.getFirstTxId(), readCompletedLogSegment.getFirstTxId());
    assertEquals(completedLogSegment.getLogSegmentId(), readCompletedLogSegment.getLogSegmentId());
    assertEquals(completedLogSegment.getLogSegmentSequenceNumber(), readCompletedLogSegment.getLogSegmentSequenceNumber());
    assertEquals(completedLogSegment.getRegionId(), readCompletedLogSegment.getRegionId());
    assertEquals(completedLogSegment.getZkPath(), readCompletedLogSegment.getZkPath());
    assertEquals(completedLogSegment.getZNodeName(), readCompletedLogSegment.getZNodeName());
    readInprogressLogSegment = segmentList.get(2L);
    assertNotNull(readInprogressLogSegment);
    assertEquals(inprogressLogSegment, readInprogressLogSegment);
}