Examples with DistributedLogManager org.apache.distributedlog.api.DistributedLogManager used on opensource projects

Example 1 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class DistributedLogAdmin method checkStream.

private static StreamCandidate checkStream(final Namespace namespace, final String streamName, final OrderedScheduler scheduler) throws IOException {
    DistributedLogManager dlm = namespace.openLog(streamName);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        if (segments.isEmpty()) {
            return null;
        }
        List<CompletableFuture<LogSegmentCandidate>> futures = new ArrayList<CompletableFuture<LogSegmentCandidate>>(segments.size());
        for (LogSegmentMetadata segment : segments) {
            futures.add(checkLogSegment(namespace, streamName, segment, scheduler));
        }
        List<LogSegmentCandidate> segmentCandidates;
        try {
            segmentCandidates = FutureUtils.result(FutureUtils.collect(futures));
        } catch (Exception e) {
            throw new IOException("Failed on checking stream " + streamName, e);
        }
        StreamCandidate streamCandidate = new StreamCandidate(streamName);
        for (LogSegmentCandidate segmentCandidate : segmentCandidates) {
            if (null != segmentCandidate) {
                streamCandidate.addLogSegmentCandidate(segmentCandidate);
            }
        }
        if (streamCandidate.segmentCandidates.isEmpty()) {
            return null;
        }
        return streamCandidate;
    } finally {
        dlm.close();
    }
}

Example 2 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class DistributedLogAdmin method fixInprogressSegmentWithLowerSequenceNumber.

/**
 * Fix inprogress segment with lower ledger sequence number.
 *
 * @param namespace
 *          dl namespace
 * @param metadataUpdater
 *          metadata updater.
 * @param streamName
 *          stream name.
 * @param verbose
 *          print verbose messages.
 * @param interactive
 *          is confirmation needed before executing actual action.
 * @throws IOException
 */
public static void fixInprogressSegmentWithLowerSequenceNumber(final Namespace namespace, final MetadataUpdater metadataUpdater, final String streamName, final boolean verbose, final boolean interactive) throws Exception {
    DistributedLogManager dlm = namespace.openLog(streamName);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        if (verbose) {
            System.out.println("LogSegments for " + streamName + " : ");
            for (LogSegmentMetadata segment : segments) {
                System.out.println(segment.getLogSegmentSequenceNumber() + "\t: " + segment);
            }
        }
        LOG.info("Get log segments for {} : {}", streamName, segments);
        // validate log segments
        long maxCompletedLogSegmentSequenceNumber = -1L;
        LogSegmentMetadata inprogressSegment = null;
        for (LogSegmentMetadata segment : segments) {
            if (!segment.isInProgress()) {
                maxCompletedLogSegmentSequenceNumber = Math.max(maxCompletedLogSegmentSequenceNumber, segment.getLogSegmentSequenceNumber());
            } else {
                // we already found an inprogress segment
                if (null != inprogressSegment) {
                    throw new DLIllegalStateException("Multiple inprogress segments found for stream " + streamName + " : " + segments);
                }
                inprogressSegment = segment;
            }
        }
        if (null == inprogressSegment || inprogressSegment.getLogSegmentSequenceNumber() > maxCompletedLogSegmentSequenceNumber) {
            // nothing to fix
            return;
        }
        final long newLogSegmentSequenceNumber = maxCompletedLogSegmentSequenceNumber + 1;
        if (interactive && !IOUtils.confirmPrompt("Confirm to fix (Y/N), Ctrl+C to break : ")) {
            return;
        }
        final LogSegmentMetadata newSegment = FutureUtils.result(metadataUpdater.changeSequenceNumber(inprogressSegment, newLogSegmentSequenceNumber));
        LOG.info("Fixed {} : {} -> {} ", new Object[] { streamName, inprogressSegment, newSegment });
        if (verbose) {
            System.out.println("Fixed " + streamName + " : " + inprogressSegment.getZNodeName() + " -> " + newSegment.getZNodeName());
            System.out.println("\t old: " + inprogressSegment);
            System.out.println("\t new: " + newSegment);
            System.out.println();
        }
    } finally {
        dlm.close();
    }
}

Example 3 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAppendOnlyStreamWriter method testBasicReadAndWriteBehavior.

@Test(timeout = 60000)
public void testBasicReadAndWriteBehavior() throws Exception {
    String name = testNames.getMethodName();
    DistributedLogManager dlmwrite = createNewDLM(conf, name);
    DistributedLogManager dlmreader = createNewDLM(conf, name);
    byte[] byteStream = DLMTestUtil.repeatString("abc", 51).getBytes();
    long txid = 1;
    AppendOnlyStreamWriter writer = dlmwrite.getAppendOnlyStreamWriter();
    writer.write(DLMTestUtil.repeatString("abc", 11).getBytes());
    writer.write(DLMTestUtil.repeatString("abc", 40).getBytes());
    writer.force(false);
    writer.close();
    AppendOnlyStreamReader reader = dlmreader.getAppendOnlyStreamReader();
    byte[] bytesIn = new byte[byteStream.length];
    int read = reader.read(bytesIn, 0, 23);
    assertEquals(23, read);
    read = reader.read(bytesIn, 23, 31);
    assertEquals(read, 31);
    byte[] bytesInTemp = new byte[byteStream.length];
    read = reader.read(bytesInTemp, 0, byteStream.length);
    assertEquals(read, byteStream.length - 23 - 31);
    read = new ByteArrayInputStream(bytesInTemp).read(bytesIn, 23 + 31, byteStream.length - 23 - 31);
    assertEquals(read, byteStream.length - 23 - 31);
    assertArrayEquals(bytesIn, byteStream);
    reader.close();
    dlmreader.close();
    dlmwrite.close();
}

Example 4 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAppendOnlyStreamWriter method testPositionUpdatesOnlyAfterWriteCompletionWithoutFsync.

@Test(timeout = 60000)
public void testPositionUpdatesOnlyAfterWriteCompletionWithoutFsync() throws Exception {
    String name = testNames.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    conf.setPeriodicFlushFrequencyMilliSeconds(1 * 1000);
    conf.setImmediateFlushEnabled(false);
    conf.setOutputBufferSize(1024 * 1024);
    DistributedLogManager dlmwriter = createNewDLM(conf, name);
    byte[] byteStream = DLMTestUtil.repeatString("abc", 11).getBytes();
    AppendOnlyStreamWriter writer = dlmwriter.getAppendOnlyStreamWriter();
    assertEquals(0, writer.position());
    Utils.ioResult(writer.write(byteStream));
    // let WriteCompleteListener have time to run
    Thread.sleep(100);
    assertEquals(33, writer.position());
    writer.close();
    dlmwriter.close();
}

Example 5 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderLock method testReaderLockBackgroundReaderLockAcquire.

@Test(timeout = 60000)
public void testReaderLockBackgroundReaderLockAcquire() throws Exception {
    final String name = runtime.getMethodName();
    DistributedLogManager dlm = createNewDLM(conf, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));
    writer.closeAndComplete();
    CompletableFuture<AsyncLogReader> futureReader1 = dlm.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    AsyncLogReader reader1 = Utils.ioResult(futureReader1);
    reader1.readNext();
    final CountDownLatch acquiredLatch = new CountDownLatch(1);
    final AtomicBoolean acquired = new AtomicBoolean(false);
    Thread acquireThread = new Thread(new Runnable() {

        @Override
        public void run() {
            CompletableFuture<AsyncLogReader> futureReader2 = null;
            DistributedLogManager dlm2 = null;
            try {
                dlm2 = createNewDLM(conf, name);
                futureReader2 = dlm2.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
                AsyncLogReader reader2 = Utils.ioResult(futureReader2);
                acquired.set(true);
                acquiredLatch.countDown();
            } catch (Exception ex) {
                fail("shouldn't reach here");
            } finally {
                try {
                    dlm2.close();
                } catch (Exception ex) {
                    fail("shouldn't reach here");
                }
            }
        }
    }, "acquire-thread");
    acquireThread.start();
    Thread.sleep(1000);
    assertEquals(false, acquired.get());
    Utils.close(reader1);
    acquiredLatch.await();
    assertEquals(true, acquired.get());
    dlm.close();
}