Examples with AsyncLogWriter org.apache.distributedlog.api.AsyncLogWriter used on opensource projects

Example 16 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestBKLogSegmentEntryReader method generateCompletedLogSegments.

void generateCompletedLogSegments(DistributedLogManager dlm, DistributedLogConfiguration conf, long numCompletedSegments, long segmentSize) throws Exception {
    long txid = 1L;
    for (long i = 0; i < numCompletedSegments; i++) {
        AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter());
        for (long j = 1; j <= segmentSize; j++) {
            Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txid++)));
            LogRecord ctrlRecord = DLMTestUtil.getLogRecordInstance(txid);
            ctrlRecord.setControl();
            Utils.ioResult(writer.write(ctrlRecord));
        }
        Utils.close(writer);
    }
}

Example 17 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestBKLogSegmentEntryReader method testReadEntriesOnStateChange.

@Test(timeout = 60000)
public void testReadEntriesOnStateChange() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(20);
    confLocal.setMaxPrefetchEntriesPerLogSegment(20);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    AsyncLogWriter writer = createInprogressLogSegment(dlm, confLocal, 5);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one", 1, segments.size());
    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();
    long expectedLastAddConfirmed = 8L;
    // wait until sending out all prefetch requests
    while (reader.readAheadEntries.size() < expectedLastAddConfirmed + 2) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals(expectedLastAddConfirmed + 2, reader.getNextEntryId());
    long txId = 1L;
    long entryId = 0L;
    while (true) {
        Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
        LogRecordWithDLSN record = entryReader.nextRecord();
        while (null != record) {
            if (!record.isControl()) {
                DLMTestUtil.verifyLogRecord(record);
                assertEquals(txId, record.getTransactionId());
                ++txId;
            }
            DLSN dlsn = record.getDlsn();
            assertEquals(1L, dlsn.getLogSegmentSequenceNo());
            assertEquals(entryId, dlsn.getEntryId());
            record = entryReader.nextRecord();
        }
        ++entryId;
        if (entryId == expectedLastAddConfirmed + 1) {
            break;
        }
    }
    assertEquals(6L, txId);
    CompletableFuture<List<Entry.Reader>> nextReadFuture = reader.readNext(1);
    // write another record to commit previous writes
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId)));
    // the long poll will be satisfied
    List<Entry.Reader> nextReadEntries = Utils.ioResult(nextReadFuture);
    assertEquals(1, nextReadEntries.size());
    Entry.Reader entryReader = nextReadEntries.get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    assertNotNull(record);
    assertTrue(record.isControl());
    assertNull(entryReader.nextRecord());
    // once the read is advanced, we will prefetch next record
    while (reader.getNextEntryId() <= entryId) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals(entryId + 2, reader.getNextEntryId());
    assertEquals(1, reader.readAheadEntries.size());
    // advance the entry id
    ++entryId;
    // close the writer, the write will be committed
    Utils.close(writer);
    entryReader = Utils.ioResult(reader.readNext(1)).get(0);
    record = entryReader.nextRecord();
    assertNotNull(record);
    assertFalse(record.isControl());
    assertNull(entryReader.nextRecord());
    while (reader.getNextEntryId() <= entryId + 1) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals(entryId + 2, reader.getNextEntryId());
    assertEquals(1, reader.readAheadEntries.size());
    // get the new log segment
    List<LogSegmentMetadata> newSegments = dlm.getLogSegments();
    assertEquals(1, newSegments.size());
    assertFalse(newSegments.get(0).isInProgress());
    reader.onLogSegmentMetadataUpdated(newSegments.get(0));
    // should be cancelled and end of log segment should be signaled correctly
    try {
        // when we closed the log segment, another control record will be
        // written, so we loop over the reader until we reach end of log segment.
        Utils.ioResult(reader.readNext(1));
        Utils.ioResult(reader.readNext(1));
        fail("Should reach end of log segment");
    } catch (EndOfLogSegmentException eol) {
    // expected
    }
    Utils.close(reader);
}

Example 18 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestBKLogSegmentEntryReader method createInprogressLogSegment.

AsyncLogWriter createInprogressLogSegment(DistributedLogManager dlm, DistributedLogConfiguration conf, long segmentSize) throws Exception {
    AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter());
    for (long i = 1L; i <= segmentSize; i++) {
        Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(i)));
        LogRecord ctrlRecord = DLMTestUtil.getLogRecordInstance(i);
        ctrlRecord.setControl();
        Utils.ioResult(writer.write(ctrlRecord));
    }
    return writer;
}

Example 19 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class DLFileSystem method append.

private FSDataOutputStream append(Path path, int bufferSize, Optional<DistributedLogConfiguration> confLocal) throws IOException {
    try {
        DistributedLogManager dlm = namespace.openLog(getStreamName(path), confLocal, Optional.empty(), Optional.empty());
        AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter());
        return new FSDataOutputStream(new BufferedOutputStream(new DLOutputStream(dlm, writer), bufferSize), statistics, writer.getLastTxId() < 0L ? 0L : writer.getLastTxId());
    } catch (LogNotFoundException le) {
        throw new FileNotFoundException(path.toString());
    }
}

Example 20 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestTruncate method testPartiallyTruncateTruncatedSegments.

@Test(timeout = 60000)
public void testPartiallyTruncateTruncatedSegments() throws Exception {
    String name = "distrlog-partially-truncate-truncated-segments";
    URI uri = createDLMURI("/" + name);
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setExplicitTruncationByApplication(true);
    // populate
    Map<Long, DLSN> dlsnMap = new HashMap<Long, DLSN>();
    populateData(dlsnMap, confLocal, name, 4, 10, false);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    LOG.info("Segments before modifying segment status : {}", segments);
    ZooKeeperClient zkc = TestZooKeeperClientBuilder.newBuilder(conf).uri(uri).build();
    for (int i = 0; i < 4; i++) {
        LogSegmentMetadata segment = segments.get(i);
        setTruncationStatus(zkc, segment, TruncationStatus.TRUNCATED);
    }
    List<LogSegmentMetadata> newSegments = dlm.getLogSegments();
    LOG.info("Segments after changing truncation status : {}", newSegments);
    dlm.close();
    DistributedLogManager newDLM = createNewDLM(confLocal, name);
    AsyncLogWriter newWriter = newDLM.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(newWriter.truncate(dlsnMap.get(15L)));
    List<LogSegmentMetadata> newSegments2 = newDLM.getLogSegments();
    assertArrayEquals(newSegments.toArray(new LogSegmentMetadata[4]), newSegments2.toArray(new LogSegmentMetadata[4]));
    Utils.close(newWriter);
    newDLM.close();
    zkc.close();
}