Examples with Versioned org.apache.bookkeeper.versioning.Versioned used on opensource projects

Example 26 with Versioned

use of org.apache.bookkeeper.versioning.Versioned in project incubator-pulsar by apache.

the class BrokerBookieIsolationTest method assertAffinityBookies.

private void assertAffinityBookies(LedgerManager ledgerManager, List<LedgerInfo> ledgers1, Set<BookieId> defaultBookies) throws Exception {
    for (LedgerInfo lInfo : ledgers1) {
        long ledgerId = lInfo.getLedgerId();
        CompletableFuture<Versioned<LedgerMetadata>> ledgerMetaFuture = ledgerManager.readLedgerMetadata(ledgerId);
        LedgerMetadata ledgerMetadata = ledgerMetaFuture.get().getValue();
        Set<BookieId> ledgerBookies = Sets.newHashSet();
        ledgerBookies.addAll(ledgerMetadata.getAllEnsembles().values().iterator().next());
        assertEquals(ledgerBookies.size(), defaultBookies.size());
        ledgerBookies.removeAll(defaultBookies);
        assertEquals(ledgerBookies.size(), 0);
    }
}

Example 27 with Versioned

use of org.apache.bookkeeper.versioning.Versioned in project incubator-pulsar by apache.

the class BrokerBookieIsolationTest method testSetRackInfoAndAffinityGroupDuringProduce.

@Test
public void testSetRackInfoAndAffinityGroupDuringProduce() throws Exception {
    final String tenant1 = "tenant1";
    final String cluster = "use";
    final String ns2 = String.format("%s/%s/%s", tenant1, cluster, "ns2");
    final int totalPublish = 100;
    final String brokerBookkeeperClientIsolationGroups = "default-group";
    final String tenantNamespaceIsolationGroups = "tenant1-isolation";
    BookieServer[] bookies = bkEnsemble.getBookies();
    ZooKeeper zkClient = bkEnsemble.getZkClient();
    Set<BookieId> isolatedBookies = Sets.newHashSet(bookies[2].getBookieId(), bookies[3].getBookieId());
    ServiceConfiguration config = new ServiceConfiguration();
    config.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config.setClusterName(cluster);
    config.setWebServicePort(Optional.of(0));
    config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config.setBrokerShutdownTimeoutMs(0L);
    config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d));
    config.setBrokerServicePort(Optional.of(0));
    config.setAdvertisedAddress("localhost");
    config.setStrictBookieAffinityEnabled(true);
    config.setBookkeeperClientIsolationGroups(brokerBookkeeperClientIsolationGroups);
    config.setManagedLedgerDefaultEnsembleSize(2);
    config.setManagedLedgerDefaultWriteQuorum(2);
    config.setManagedLedgerDefaultAckQuorum(2);
    config.setAllowAutoTopicCreationType("non-partitioned");
    int totalEntriesPerLedger = 20;
    int totalLedgers = totalPublish / totalEntriesPerLedger;
    config.setManagedLedgerMaxEntriesPerLedger(totalEntriesPerLedger);
    config.setManagedLedgerMinLedgerRolloverTimeMinutes(0);
    pulsarService = new PulsarService(config);
    pulsarService.start();
    PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarService.getWebServiceAddress()).build();
    ClusterData clusterData = ClusterData.builder().serviceUrl(pulsarService.getWebServiceAddress()).build();
    admin.clusters().createCluster(cluster, clusterData);
    TenantInfoImpl tenantInfo = new TenantInfoImpl(null, Sets.newHashSet(cluster));
    admin.tenants().createTenant(tenant1, tenantInfo);
    admin.namespaces().createNamespace(ns2);
    try {
        admin.namespaces().getBookieAffinityGroup(ns2);
        fail("ns2 should have no bookie affinity group set");
    } catch (PulsarAdminException.NotFoundException e) {
    // Ok
    }
    @Cleanup PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsarService.getBrokerServiceUrl()).statsInterval(-1, TimeUnit.SECONDS).build();
    final String topicName = String.format("persistent://%s/%s", ns2, "topic1");
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscribe();
    consumer.close();
    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer().topic(topicName).sendTimeout(5, TimeUnit.SECONDS);
    Producer<byte[]> producer = producerBuilder.create();
    for (int i = 0; i < 20; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    setDefaultIsolationGroup(tenantNamespaceIsolationGroups, zkClient, isolatedBookies);
    admin.namespaces().setBookieAffinityGroup(ns2, BookieAffinityGroupData.builder().bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroups).build());
    assertEquals(admin.namespaces().getBookieAffinityGroup(ns2), BookieAffinityGroupData.builder().bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroups).build());
    PersistentTopic topic2 = (PersistentTopic) pulsarService.getBrokerService().getTopicReference(topicName).get();
    ManagedLedgerImpl ml = (ManagedLedgerImpl) topic2.getManagedLedger();
    Awaitility.await().untilAsserted(() -> Assert.assertTrue(ml.getConfig().getBookKeeperEnsemblePlacementPolicyProperties().size() > 0));
    for (int i = 0; i < 80; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    producer.close();
    BookieImpl bookie1 = (BookieImpl) bookies[0].getBookie();
    LedgerManager ledgerManager = getLedgerManager(bookie1);
    ManagedLedgerImpl ml2 = (ManagedLedgerImpl) topic2.getManagedLedger();
    // namespace: ns2
    assertEquals(ml2.getLedgersInfoAsList().size(), totalLedgers);
    List<LedgerInfo> ledgers = ml2.getLedgersInfoAsList();
    // validate ledgers' ensemble with affinity bookies
    for (int i = 1; i < ledgers.size(); i++) {
        LedgerInfo lInfo = ledgers.get(i);
        long ledgerId = lInfo.getLedgerId();
        CompletableFuture<Versioned<LedgerMetadata>> ledgerMetaFuture = ledgerManager.readLedgerMetadata(ledgerId);
        LedgerMetadata ledgerMetadata = ledgerMetaFuture.get().getValue();
        Set<BookieId> ledgerBookies = Sets.newHashSet();
        ledgerBookies.addAll(ledgerMetadata.getAllEnsembles().values().iterator().next());
        assertEquals(ledgerBookies.size(), isolatedBookies.size());
        ledgerBookies.removeAll(isolatedBookies);
        assertEquals(ledgerBookies.size(), 0);
    }
}

Example 28 with Versioned

use of org.apache.bookkeeper.versioning.Versioned in project bookkeeper by apache.

the class MockLedgerManager method writeLedgerMetadata.

@Override
public CompletableFuture<Versioned<LedgerMetadata>> writeLedgerMetadata(long ledgerId, LedgerMetadata metadata, Version currentVersion) {
    CompletableFuture<Versioned<LedgerMetadata>> promise = new CompletableFuture<>();
    preWriteHook.runHook(ledgerId, metadata).thenComposeAsync((ignore) -> {
        try {
            Versioned<LedgerMetadata> oldMetadata = readMetadata(ledgerId);
            if (oldMetadata == null) {
                return FutureUtils.exception(new BKException.BKNoSuchLedgerExistsOnMetadataServerException());
            } else if (!oldMetadata.getVersion().equals(currentVersion)) {
                return FutureUtils.exception(new BKException.BKMetadataVersionException());
            } else {
                LongVersion oldVersion = (LongVersion) oldMetadata.getVersion();
                metadataMap.put(ledgerId, Pair.of(new LongVersion(oldVersion.getLongVersion() + 1), serDe.serialize(metadata)));
                Versioned<LedgerMetadata> readBack = readMetadata(ledgerId);
                return FutureUtils.value(readBack);
            }
        } catch (Exception e) {
            LOG.error("Error writing metadata", e);
            return FutureUtils.exception(e);
        }
    }, executor).whenComplete((res, ex) -> {
        if (ex != null) {
            Throwable cause = (ex instanceof CompletionException) ? ex.getCause() : ex;
            executeCallback(() -> promise.completeExceptionally(cause));
        } else {
            executeCallback(() -> promise.complete(res));
        }
    });
    return promise;
}

Example 29 with Versioned

use of org.apache.bookkeeper.versioning.Versioned in project bookkeeper by apache.

the class GcLedgersTest method testGcLedgersIfReadLedgerMetadataFailsForDeletedLedgers.

/*
     * In this test scenario all the created ledgers are deleted, but LedgerManager.readLedgerMetadata fails with
     * ZKException. So even in this case, GarbageCollector.gc shouldn't delete ledgers data.
     *
     * ScanAndCompareGarbageCollector/GC should clean data of ledger only if both the LedgerManager.getLedgerRanges says
     * that ledger is not existing and also ledgerManager.readLedgerMetadata fails with error
     * NoSuchLedgerExistsOnMetadataServerException, but is shouldn't delete if the readLedgerMetadata fails with any
     * other error.
     */
@Test
public void testGcLedgersIfReadLedgerMetadataFailsForDeletedLedgers() throws Exception {
    baseConf.setVerifyMetadataOnGc(true);
    final SortedSet<Long> createdLedgers = Collections.synchronizedSortedSet(new TreeSet<Long>());
    final SortedSet<Long> cleaned = Collections.synchronizedSortedSet(new TreeSet<Long>());
    // Create few ledgers
    final int numLedgers = 5;
    createLedgers(numLedgers, createdLedgers);
    CompletableFuture<Versioned<LedgerMetadata>> errorFuture = new CompletableFuture<>();
    errorFuture.completeExceptionally(new BKException.ZKException());
    LedgerManager mockLedgerManager = new CleanupLedgerManager(getLedgerManager()) {

        @Override
        public CompletableFuture<Versioned<LedgerMetadata>> readLedgerMetadata(long ledgerId) {
            return errorFuture;
        }
    };
    final GarbageCollector garbageCollector = new ScanAndCompareGarbageCollector(mockLedgerManager, new MockLedgerStorage(), baseConf, NullStatsLogger.INSTANCE);
    GarbageCollector.GarbageCleaner cleaner = new GarbageCollector.GarbageCleaner() {

        @Override
        public void clean(long ledgerId) {
            LOG.info("Cleaned {}", ledgerId);
            cleaned.add(ledgerId);
        }
    };
    validateLedgerRangeIterator(createdLedgers);
    for (long ledgerId : createdLedgers) {
        removeLedger(ledgerId);
    }
    garbageCollector.gc(cleaner);
    assertTrue("Should have cleaned nothing", cleaned.isEmpty());
}

Example 30 with Versioned

use of org.apache.bookkeeper.versioning.Versioned in project bookkeeper by apache.

the class ZKRegistrationManager method readCookie.

@Override
public Versioned<byte[]> readCookie(BookieId bookieId) throws BookieException {
    String zkPath = getCookiePath(bookieId);
    try {
        Stat stat = zk.exists(zkPath, false);
        byte[] data = zk.getData(zkPath, false, stat);
        // sets stat version from ZooKeeper
        LongVersion version = new LongVersion(stat.getVersion());
        return new Versioned<>(data, version);
    } catch (NoNodeException nne) {
        throw new CookieNotFoundException(bookieId.toString());
    } catch (KeeperException | InterruptedException e) {
        throw new MetadataStoreException("Failed to read cookie for bookie " + bookieId);
    }
}