Examples with ManagedLedgerConfig org.apache.bookkeeper.mledger.ManagedLedgerConfig used on opensource projects

Example 16 with ManagedLedgerConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerConfig in project pulsar by yahoo.

the class ManagedCursorTest method testRateLimitMarkDelete.

@Test(timeOut = 20000)
void testRateLimitMarkDelete() throws Exception {
    ManagedLedgerConfig config = new ManagedLedgerConfig();
    // Throttle to 1/s
    config.setThrottleMarkDelete(1);
    ManagedLedger ledger = factory.open("my_test_ledger", config);
    ManagedCursor c1 = ledger.openCursor("c1");
    Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
    Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
    Position p3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding));
    assertEquals(c1.getNumberOfEntriesInBacklog(), 3);
    c1.markDelete(p1);
    c1.markDelete(p2);
    c1.markDelete(p3);
    assertEquals(c1.getNumberOfEntriesInBacklog(), 0);
    // Re-open to recover from storage
    ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
    ledger = factory2.open("my_test_ledger", new ManagedLedgerConfig());
    c1 = ledger.openCursor("c1");
    // Only the 1st mark-delete was persisted
    assertEquals(c1.getNumberOfEntriesInBacklog(), 2);
    factory2.shutdown();
}

Example 17 with ManagedLedgerConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerConfig in project pulsar by yahoo.

the class PersistentTopicConcurrentTest method setup.

@BeforeMethod
public void setup(Method m) throws Exception {
    super.setUp(m);
    ServiceConfiguration svcConfig = spy(new ServiceConfiguration());
    PulsarService pulsar = spy(new PulsarService(svcConfig));
    doReturn(svcConfig).when(pulsar).getConfiguration();
    mlFactoryMock = mock(ManagedLedgerFactory.class);
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
    ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
    final ManagedCursor cursor = ledger.openCursor("c1");
    cursorMock = cursor;
    ledgerMock = ledger;
    mlFactoryMock = factory;
    doReturn(mlFactoryMock).when(pulsar).getManagedLedgerFactory();
    ZooKeeper mockZk = mock(ZooKeeper.class);
    doReturn(mockZk).when(pulsar).getZkClient();
    brokerService = spy(new BrokerService(pulsar));
    doReturn(brokerService).when(pulsar).getBrokerService();
    serverCnx = spy(new ServerCnx(brokerService));
    doReturn(true).when(serverCnx).isActive();
    NamespaceService nsSvc = mock(NamespaceService.class);
    doReturn(nsSvc).when(pulsar).getNamespaceService();
    doReturn(true).when(nsSvc).isServiceUnitOwned(any(NamespaceBundle.class));
    doReturn(true).when(nsSvc).isServiceUnitActive(any(DestinationName.class));
    final List<Position> addedEntries = Lists.newArrayList();
    for (int i = 0; i < 100; i++) {
        Position pos = ledger.addEntry("entry".getBytes());
        addedEntries.add(pos);
    }
}

Example 18 with ManagedLedgerConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerConfig in project pulsar by yahoo.

the class BrokerService method getManagedLedgerConfig.

public CompletableFuture<ManagedLedgerConfig> getManagedLedgerConfig(DestinationName topicName) {
    CompletableFuture<ManagedLedgerConfig> future = new CompletableFuture<>();
    // Execute in background thread, since getting the policies might block if the z-node wasn't already cached
    pulsar.getOrderedExecutor().submitOrdered(topicName, safeRun(() -> {
        NamespaceName namespace = topicName.getNamespaceObject();
        ServiceConfiguration serviceConfig = pulsar.getConfiguration();
        // Get persistence policy for this destination
        Policies policies;
        try {
            policies = pulsar.getConfigurationCache().policiesCache().get(AdminResource.path("policies", namespace.getProperty(), namespace.getCluster(), namespace.getLocalName())).orElse(null);
        } catch (Throwable t) {
            // Ignoring since if we don't have policies, we fallback on the default
            log.warn("Got exception when reading persistence policy for {}: {}", topicName, t.getMessage(), t);
            future.completeExceptionally(t);
            return;
        }
        PersistencePolicies persistencePolicies = policies != null ? policies.persistence : null;
        RetentionPolicies retentionPolicies = policies != null ? policies.retention_policies : null;
        if (persistencePolicies == null) {
            // Apply default values
            persistencePolicies = new PersistencePolicies(serviceConfig.getManagedLedgerDefaultEnsembleSize(), serviceConfig.getManagedLedgerDefaultWriteQuorum(), serviceConfig.getManagedLedgerDefaultAckQuorum(), serviceConfig.getManagedLedgerDefaultMarkDeleteRateLimit());
        }
        if (retentionPolicies == null) {
            retentionPolicies = new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), serviceConfig.getDefaultRetentionSizeInMB());
        }
        ManagedLedgerConfig config = new ManagedLedgerConfig();
        config.setEnsembleSize(persistencePolicies.getBookkeeperEnsemble());
        config.setWriteQuorumSize(persistencePolicies.getBookkeeperWriteQuorum());
        config.setAckQuorumSize(persistencePolicies.getBookkeeperAckQuorum());
        config.setThrottleMarkDelete(persistencePolicies.getManagedLedgerMaxMarkDeleteRate());
        config.setDigestType(DigestType.CRC32);
        config.setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist());
        config.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger());
        config.setMinimumRolloverTime(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(), TimeUnit.MINUTES);
        config.setMaximumRolloverTime(serviceConfig.getManagedLedgerMaxLedgerRolloverTimeMinutes(), TimeUnit.MINUTES);
        config.setMaxSizePerLedgerMb(2048);
        config.setMetadataEnsembleSize(serviceConfig.getManagedLedgerDefaultEnsembleSize());
        config.setMetadataWriteQuorumSize(serviceConfig.getManagedLedgerDefaultWriteQuorum());
        config.setMetadataAckQuorumSize(serviceConfig.getManagedLedgerDefaultAckQuorum());
        config.setMetadataMaxEntriesPerLedger(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger());
        config.setLedgerRolloverTimeout(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds());
        config.setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES);
        config.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB());
        future.complete(config);
    }, (exception) -> future.completeExceptionally(exception)));
    return future;
}

Example 19 with ManagedLedgerConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerConfig in project pulsar by yahoo.

the class PersistentTopics method getBacklog.

@GET
@Path("{property}/{cluster}/{namespace}/{destination}/backlog")
@ApiOperation(value = "Get estimated backlog for offline topic.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") })
public PersistentOfflineTopicStats getBacklog(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("destination") @Encoded String destination, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
    destination = decode(destination);
    validateAdminAccessOnProperty(property);
    // note that we do not want to load the topic and hence skip validateAdminOperationOnDestination()
    try {
        policiesCache().get(path("policies", property, cluster, namespace));
    } catch (KeeperException.NoNodeException e) {
        log.warn("[{}] Failed to get topic backlog {}/{}/{}: Namespace does not exist", clientAppId(), property, cluster, namespace);
        throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
    } catch (Exception e) {
        log.error("[{}] Failed to get topic backlog {}/{}/{}", clientAppId(), property, cluster, namespace, e);
        throw new RestException(e);
    }
    DestinationName dn = DestinationName.get(domain(), property, cluster, namespace, destination);
    PersistentOfflineTopicStats offlineTopicStats = null;
    try {
        offlineTopicStats = pulsar().getBrokerService().getOfflineTopicStat(dn);
        if (offlineTopicStats != null) {
            // offline topic stat has a cost - so use cached value until TTL
            long elapsedMs = System.currentTimeMillis() - offlineTopicStats.statGeneratedAt.getTime();
            if (TimeUnit.MINUTES.convert(elapsedMs, TimeUnit.MILLISECONDS) < OFFLINE_TOPIC_STAT_TTL_MINS) {
                return offlineTopicStats;
            }
        }
        final ManagedLedgerConfig config = pulsar().getBrokerService().getManagedLedgerConfig(dn).get();
        ManagedLedgerOfflineBacklog offlineTopicBacklog = new ManagedLedgerOfflineBacklog(config.getDigestType(), config.getPassword(), pulsar().getAdvertisedAddress(), false);
        offlineTopicStats = offlineTopicBacklog.estimateUnloadedTopicBacklog((ManagedLedgerFactoryImpl) pulsar().getManagedLedgerFactory(), dn);
        pulsar().getBrokerService().cacheOfflineTopicStats(dn, offlineTopicStats);
    } catch (Exception exception) {
        throw new RestException(exception);
    }
    return offlineTopicStats;
}

Example 20 with ManagedLedgerConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerConfig in project pulsar by yahoo.

the class ManagedLedgerBkTest method testConcurrentMarkDelete.

@Test
public void testConcurrentMarkDelete() throws Exception {
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
    ManagedLedgerConfig mlConfig = new ManagedLedgerConfig();
    mlConfig.setEnsembleSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataAckQuorumSize(1);
    // set the data ledger size
    mlConfig.setMaxEntriesPerLedger(100);
    // set the metadata ledger size to 1 to kick off many ledger switching cases
    mlConfig.setMetadataMaxEntriesPerLedger(10);
    ManagedLedger ledger = factory.open("ml-markdelete-ledger", mlConfig);
    final List<Position> addedEntries = Lists.newArrayList();
    int numCursors = 10;
    final CyclicBarrier barrier = new CyclicBarrier(numCursors);
    List<ManagedCursor> cursors = Lists.newArrayList();
    for (int i = 0; i < numCursors; i++) {
        cursors.add(ledger.openCursor(String.format("c%d", i)));
    }
    for (int i = 0; i < 50; i++) {
        Position pos = ledger.addEntry("entry".getBytes());
        addedEntries.add(pos);
    }
    List<Future<?>> futures = Lists.newArrayList();
    for (ManagedCursor cursor : cursors) {
        futures.add(executor.submit(() -> {
            barrier.await();
            for (Position position : addedEntries) {
                cursor.markDelete(position);
            }
            return null;
        }));
    }
    for (Future<?> future : futures) {
        future.get();
    }
    // Since in this test we roll-over the cursor ledger every 10 entries acknowledged, the background roll back
    // might still be happening when the futures are completed.
    Thread.sleep(1000);
    factory.shutdown();
}