Examples with Namespace org.apache.distributedlog.api.namespace.Namespace used on opensource projects

Example 36 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project bookkeeper by apache.

the class TestLogSegmentsZK method testCreateLogSegmentUnmatchMaxSequenceNumber.

/**
 * Create Log Segment while max sequence number isn't match with list of log segments.
 */
@Test(timeout = 60000)
public void testCreateLogSegmentUnmatchMaxSequenceNumber() throws Exception {
    URI uri = createURI();
    String streamName = testName.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration().setLockTimeout(99999).setOutputBufferSize(0).setImmediateFlushEnabled(true).setEnableLedgerAllocatorPool(true).setLedgerAllocatorPoolName("test");
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    namespace.createLog(streamName);
    MaxLogSegmentSequenceNo max1 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf);
    assertEquals(DistributedLogConstants.UNASSIGNED_LOGSEGMENT_SEQNO, max1.getSequenceNumber());
    DistributedLogManager dlm = namespace.openLog(streamName);
    final int numSegments = 3;
    for (int i = 0; i < numSegments; i++) {
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        out.write(DLMTestUtil.getLogRecordInstance(i));
        out.closeAndComplete();
    }
    MaxLogSegmentSequenceNo max2 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf);
    assertEquals(3, max2.getSequenceNumber());
    // update the max ledger sequence number
    updateMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf, DLUtils.serializeLogSegmentSequenceNumber(99));
    DistributedLogManager dlm1 = namespace.openLog(streamName);
    try {
        BKSyncLogWriter out1 = (BKSyncLogWriter) dlm1.startLogSegmentNonPartitioned();
        out1.write(DLMTestUtil.getLogRecordInstance(numSegments + 1));
        out1.closeAndComplete();
        fail("Should fail creating new log segment when encountered unmatch max ledger sequence number");
    } catch (DLIllegalStateException lse) {
    // expected
    } finally {
        dlm1.close();
    }
    DistributedLogManager dlm2 = namespace.openLog(streamName);
    List<LogSegmentMetadata> segments = dlm2.getLogSegments();
    try {
        assertEquals(3, segments.size());
        assertEquals(1L, segments.get(0).getLogSegmentSequenceNumber());
        assertEquals(2L, segments.get(1).getLogSegmentSequenceNumber());
        assertEquals(3L, segments.get(2).getLogSegmentSequenceNumber());
    } finally {
        dlm2.close();
    }
    dlm.close();
    namespace.close();
}

Example 37 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project bookkeeper by apache.

the class TestLogSegmentsZK method testCompleteLogSegmentConflicts.

@Test(timeout = 60000)
public void testCompleteLogSegmentConflicts() throws Exception {
    URI uri = createURI();
    String streamName = testName.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration().setLockTimeout(99999).setOutputBufferSize(0).setImmediateFlushEnabled(true).setEnableLedgerAllocatorPool(true).setLedgerAllocatorPoolName("test");
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    namespace.createLog(streamName);
    DistributedLogManager dlm1 = namespace.openLog(streamName);
    DistributedLogManager dlm2 = namespace.openLog(streamName);
    // dlm1 is writing
    BKSyncLogWriter out1 = (BKSyncLogWriter) dlm1.startLogSegmentNonPartitioned();
    out1.write(DLMTestUtil.getLogRecordInstance(1));
    // before out1 complete, out2 is in on recovery
    // it completed the log segments which bump the version of /ledgers znode
    BKAsyncLogWriter out2 = (BKAsyncLogWriter) dlm2.startAsyncLogSegmentNonPartitioned();
    try {
        out1.closeAndComplete();
        fail("Should fail closeAndComplete since other people already completed it.");
    } catch (IOException ioe) {
    }
}

Example 38 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project bookkeeper by apache.

the class TestLogSegmentCreation method testCreateLogSegmentAfterLoseLock.

@Test(timeout = 60000)
public void testCreateLogSegmentAfterLoseLock() throws Exception {
    URI uri = createDLMURI("/LogSegmentCreation");
    String name = "distrlog-createlogsegment-afterloselock";
    DistributedLogConfiguration conf = new DistributedLogConfiguration().setLockTimeout(99999).setOutputBufferSize(0).setImmediateFlushEnabled(true).setEnableLedgerAllocatorPool(true).setLedgerAllocatorPoolName("test");
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    DistributedLogManager dlm = namespace.openLog(name);
    final int numSegments = 3;
    for (int i = 0; i < numSegments; i++) {
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        out.write(DLMTestUtil.getLogRecordInstance(i));
        out.closeAndComplete();
    }
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    LOG.info("Segments : {}", segments);
    assertEquals(3, segments.size());
    final DistributedLogManager dlm1 = namespace.openLog(name);
    final DistributedLogManager dlm2 = namespace.openLog(name);
    BKAsyncLogWriter writer1 = (BKAsyncLogWriter) dlm1.startAsyncLogSegmentNonPartitioned();
    LOG.info("Created writer 1.");
    BKSyncLogWriter writer2 = (BKSyncLogWriter) dlm2.startLogSegmentNonPartitioned();
    LOG.info("Created writer 2.");
    writer2.write(DLMTestUtil.getLogRecordInstance(numSegments));
    writer2.closeAndComplete();
    try {
        Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(numSegments + 1)));
        fail("Should fail on writing new log records.");
    } catch (Throwable t) {
        LOG.error("Failed to write entry : ", t);
    }
    segments = dlm.getLogSegments();
    boolean hasInprogress = false;
    boolean hasDuplicatedSegment = false;
    long nextSeqNo = segments.get(0).getLogSegmentSequenceNumber();
    for (int i = 1; i < segments.size(); i++) {
        LogSegmentMetadata segment = segments.get(i);
        assertTrue(segment.getLogSegmentSequenceNumber() >= nextSeqNo);
        if (segment.getLogSegmentSequenceNumber() == nextSeqNo) {
            hasDuplicatedSegment = true;
        }
        nextSeqNo = segment.getLogSegmentSequenceNumber();
        if (segment.isInProgress()) {
            hasInprogress = true;
        }
    }
    assertEquals(4, segments.size());
    assertFalse(hasInprogress);
    assertFalse(hasDuplicatedSegment);
    LOG.info("Segments : duplicated = {}, inprogress = {}, {}", new Object[] { hasDuplicatedSegment, hasInprogress, segments });
    dlm1.close();
    dlm2.close();
    dlm.close();
    namespace.close();
}

Example 39 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project bookkeeper by apache.

the class DLAuditor method calculateStreamSpaceUsage.

/**
 * Calculating stream space usage from given <i>uri</i>.
 *
 * @param uri dl uri
 * @throws IOException
 */
public Map<String, Long> calculateStreamSpaceUsage(final URI uri) throws IOException {
    logger.info("Collecting stream space usage for {}.", uri);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    try {
        return calculateStreamSpaceUsage(uri, namespace);
    } finally {
        namespace.close();
    }
}

Example 40 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project bookkeeper by apache.

the class DLAuditor method collectLedgersFromDL.

/**
 * Find leak ledgers phase 2: collect ledgers from uris.
 */
private Set<Long> collectLedgersFromDL(List<URI> uris, List<List<String>> allocationPaths) throws IOException {
    final Set<Long> ledgers = new TreeSet<Long>();
    List<Namespace> namespaces = new ArrayList<Namespace>(uris.size());
    try {
        for (URI uri : uris) {
            namespaces.add(NamespaceBuilder.newBuilder().conf(conf).uri(uri).build());
        }
        final CountDownLatch doneLatch = new CountDownLatch(uris.size());
        final AtomicInteger numFailures = new AtomicInteger(0);
        ExecutorService executor = Executors.newFixedThreadPool(uris.size());
        try {
            int i = 0;
            for (final Namespace namespace : namespaces) {
                final Namespace dlNamespace = namespace;
                final URI uri = uris.get(i);
                final List<String> aps = allocationPaths.get(i);
                i++;
                executor.submit(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            logger.info("Collecting ledgers from {} : {}", uri, aps);
                            collectLedgersFromAllocator(uri, namespace, aps, ledgers);
                            synchronized (ledgers) {
                                logger.info("Collected {} ledgers from allocators for {} : {} ", new Object[] { ledgers.size(), uri, ledgers });
                            }
                            collectLedgersFromDL(uri, namespace, ledgers);
                        } catch (IOException e) {
                            numFailures.incrementAndGet();
                            logger.info("Error to collect ledgers from DL : ", e);
                        }
                        doneLatch.countDown();
                    }
                });
            }
            try {
                doneLatch.await();
                if (numFailures.get() > 0) {
                    throw new IOException(numFailures.get() + " errors to collect ledgers from DL");
                }
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                logger.warn("Interrupted on collecting ledgers from DL : ", e);
                throw new DLInterruptedException("Interrupted on collecting ledgers from DL : ", e);
            }
        } finally {
            executor.shutdown();
        }
    } finally {
        for (Namespace namespace : namespaces) {
            namespace.close();
        }
    }
    return ledgers;
}