Examples with LogRecordWithDLSN org.apache.distributedlog.LogRecordWithDLSN used on opensource projects

Example 6 with LogRecordWithDLSN

use of org.apache.distributedlog.LogRecordWithDLSN in project bookkeeper by apache.

the class TestDistributedLogAdmin method testChangeSequenceNumber.

/**
 * {@link https://issues.apache.org/jira/browse/DL-44}.
 */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testChangeSequenceNumber() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    confLocal.setLogSegmentCacheEnabled(false);
    DistributedLogConfiguration readConf = new DistributedLogConfiguration();
    readConf.addConfiguration(conf);
    readConf.setLogSegmentCacheEnabled(false);
    readConf.setLogSegmentSequenceNumberValidationEnabled(true);
    URI uri = createDLMURI("/change-sequence-number");
    zooKeeperClient.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    Namespace readNamespace = NamespaceBuilder.newBuilder().conf(readConf).uri(uri).build();
    String streamName = "change-sequence-number";
    // create completed log segments
    DistributedLogManager dlm = namespace.openLog(streamName);
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 4, 10);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 5, 41, false, 10, true);
    dlm.close();
    // create a reader
    DistributedLogManager readDLM = readNamespace.openLog(streamName);
    AsyncLogReader reader = readDLM.getAsyncLogReader(DLSN.InitialDLSN);
    // read the records
    long expectedTxId = 1L;
    DLSN lastDLSN = DLSN.InitialDLSN;
    for (int i = 0; i < 4 * 10; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
        lastDLSN = record.getDlsn();
    }
    LOG.info("Injecting bad log segment '3'");
    dlm = namespace.openLog(streamName);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 3L, 5 * 10 + 1, true, 10, false);
    LOG.info("Injected bad log segment '3'");
    // there isn't records should be read
    CompletableFuture<LogRecordWithDLSN> readFuture = reader.readNext();
    try {
        LogRecordWithDLSN record = Utils.ioResult(readFuture);
        fail("Should fail reading next record " + record + " when there is a corrupted log segment");
    } catch (UnexpectedException ue) {
    // expected
    }
    LOG.info("Dryrun fix inprogress segment that has lower sequence number");
    // Dryrun
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(namespace, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(namespace)), streamName, false, false);
    try {
        reader = readDLM.getAsyncLogReader(lastDLSN);
        Utils.ioResult(reader.readNext());
        fail("Should fail reading next when there is a corrupted log segment");
    } catch (UnexpectedException ue) {
    // expected
    }
    LOG.info("Actual run fix inprogress segment that has lower sequence number");
    // Actual run
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(namespace, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(namespace)), streamName, false, false);
    // be able to read more after fix
    reader = readDLM.getAsyncLogReader(lastDLSN);
    // skip the first record
    Utils.ioResult(reader.readNext());
    readFuture = reader.readNext();
    expectedTxId = 51L;
    LogRecord record = Utils.ioResult(readFuture);
    assertNotNull(record);
    DLMTestUtil.verifyLogRecord(record);
    assertEquals(expectedTxId, record.getTransactionId());
    expectedTxId++;
    for (int i = 1; i < 10; i++) {
        record = Utils.ioResult(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
    }
    Utils.close(reader);
    readDLM.close();
    dlm.close();
    namespace.close();
    readNamespace.close();
}

Example 7 with LogRecordWithDLSN

use of org.apache.distributedlog.LogRecordWithDLSN in project incubator-pulsar by apache.

the class DLInputStreamTest method testRead.

/**
 * Test Case: read records from the input stream.
 */
@Test
public void testRead() throws Exception {
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    LogReader reader = mock(LogReader.class);
    when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);
    byte[] data = "test-read".getBytes(UTF_8);
    LogRecordWithDLSN record = mock(LogRecordWithDLSN.class);
    when(record.getPayLoadInputStream()).thenReturn(new ByteArrayInputStream(data));
    when(reader.readNext(anyBoolean())).thenReturn(record).thenThrow(new EndOfStreamException("eos"));
    DLInputStream in = new DLInputStream(dlm);
    int numReads = 0;
    int readByte;
    while ((readByte = in.read()) != -1) {
        assertEquals(data[numReads], readByte);
        ++numReads;
    }
    assertEquals(data.length, numReads);
}

Example 8 with LogRecordWithDLSN

use of org.apache.distributedlog.LogRecordWithDLSN in project incubator-heron by apache.

the class DLInputStreamTest method testRead.

/**
 * Test Case: read records from the input stream.
 */
@Test
public void testRead() throws Exception {
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    LogReader reader = mock(LogReader.class);
    when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);
    byte[] data = "test-read".getBytes(UTF_8);
    LogRecordWithDLSN record = mock(LogRecordWithDLSN.class);
    when(record.getPayLoadInputStream()).thenReturn(new ByteArrayInputStream(data));
    when(reader.readNext(anyBoolean())).thenReturn(record).thenThrow(new EndOfStreamException("eos"));
    DLInputStream in = new DLInputStream(dlm);
    int numReads = 0;
    int readByte;
    while ((readByte = in.read()) != -1) {
        assertEquals(data[numReads], readByte);
        ++numReads;
    }
    assertEquals(data.length, numReads);
}

Example 9 with LogRecordWithDLSN

use of org.apache.distributedlog.LogRecordWithDLSN in project bookkeeper by apache.

the class TestBKLogSegmentEntryReader method testReadEntriesOnStateChange.

@Test(timeout = 60000)
public void testReadEntriesOnStateChange() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(20);
    confLocal.setMaxPrefetchEntriesPerLogSegment(20);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    AsyncLogWriter writer = createInprogressLogSegment(dlm, confLocal, 5);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one", 1, segments.size());
    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();
    long expectedLastAddConfirmed = 8L;
    // wait until sending out all prefetch requests
    while (reader.readAheadEntries.size() < expectedLastAddConfirmed + 2) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals(expectedLastAddConfirmed + 2, reader.getNextEntryId());
    long txId = 1L;
    long entryId = 0L;
    while (true) {
        Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
        LogRecordWithDLSN record = entryReader.nextRecord();
        while (null != record) {
            if (!record.isControl()) {
                DLMTestUtil.verifyLogRecord(record);
                assertEquals(txId, record.getTransactionId());
                ++txId;
            }
            DLSN dlsn = record.getDlsn();
            assertEquals(1L, dlsn.getLogSegmentSequenceNo());
            assertEquals(entryId, dlsn.getEntryId());
            record = entryReader.nextRecord();
        }
        ++entryId;
        if (entryId == expectedLastAddConfirmed + 1) {
            break;
        }
    }
    assertEquals(6L, txId);
    CompletableFuture<List<Entry.Reader>> nextReadFuture = reader.readNext(1);
    // write another record to commit previous writes
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId)));
    // the long poll will be satisfied
    List<Entry.Reader> nextReadEntries = Utils.ioResult(nextReadFuture);
    assertEquals(1, nextReadEntries.size());
    Entry.Reader entryReader = nextReadEntries.get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    assertNotNull(record);
    assertTrue(record.isControl());
    assertNull(entryReader.nextRecord());
    // once the read is advanced, we will prefetch next record
    while (reader.getNextEntryId() <= entryId) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals(entryId + 2, reader.getNextEntryId());
    assertEquals(1, reader.readAheadEntries.size());
    // advance the entry id
    ++entryId;
    // close the writer, the write will be committed
    Utils.close(writer);
    entryReader = Utils.ioResult(reader.readNext(1)).get(0);
    record = entryReader.nextRecord();
    assertNotNull(record);
    assertFalse(record.isControl());
    assertNull(entryReader.nextRecord());
    while (reader.getNextEntryId() <= entryId + 1) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals(entryId + 2, reader.getNextEntryId());
    assertEquals(1, reader.readAheadEntries.size());
    // get the new log segment
    List<LogSegmentMetadata> newSegments = dlm.getLogSegments();
    assertEquals(1, newSegments.size());
    assertFalse(newSegments.get(0).isInProgress());
    reader.onLogSegmentMetadataUpdated(newSegments.get(0));
    // should be cancelled and end of log segment should be signaled correctly
    try {
        // when we closed the log segment, another control record will be
        // written, so we loop over the reader until we reach end of log segment.
        Utils.ioResult(reader.readNext(1));
        Utils.ioResult(reader.readNext(1));
        fail("Should reach end of log segment");
    } catch (EndOfLogSegmentException eol) {
    // expected
    }
    Utils.close(reader);
}

Example 10 with LogRecordWithDLSN

use of org.apache.distributedlog.LogRecordWithDLSN in project bookkeeper by apache.

the class TestBKLogSegmentEntryReader method testMaxPrefetchEntriesSmallSegment.

@Test(timeout = 60000)
public void testMaxPrefetchEntriesSmallSegment() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(20);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 5);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one", 1, segments.size());
    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();
    // wait for the read ahead entries to become available
    while (reader.readAheadEntries.size() < (reader.getLastAddConfirmed() + 1)) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    long txId = 1L;
    long entryId = 0L;
    assertEquals((reader.getLastAddConfirmed() + 1), reader.readAheadEntries.size());
    assertEquals((reader.getLastAddConfirmed() + 1), reader.getNextEntryId());
    // read first entry
    Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    while (null != record) {
        if (!record.isControl()) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(txId, record.getTransactionId());
            ++txId;
        }
        DLSN dlsn = record.getDlsn();
        assertEquals(1L, dlsn.getLogSegmentSequenceNo());
        assertEquals(entryId, dlsn.getEntryId());
        record = entryReader.nextRecord();
    }
    ++entryId;
    assertEquals(2L, txId);
    assertEquals(reader.getLastAddConfirmed(), reader.readAheadEntries.size());
    assertEquals((reader.getLastAddConfirmed() + 1), reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());
    Utils.close(reader);
}