Examples with MetadataStoreExtended org.apache.pulsar.metadata.api.extended.MetadataStoreExtended used on opensource projects

Example 1 with MetadataStoreExtended

use of org.apache.pulsar.metadata.api.extended.MetadataStoreExtended in project pulsar by apache.

the class PulsarTransactionCoordinatorMetadataSetup method main.

public static void main(String[] args) throws Exception {
    Arguments arguments = new Arguments();
    JCommander jcommander = new JCommander();
    try {
        jcommander.addObject(arguments);
        jcommander.parse(args);
        if (arguments.help) {
            jcommander.usage();
            return;
        }
        if (arguments.generateDocs) {
            CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar");
            cmd.addCommand("initialize-transaction-coordinator-metadata", arguments);
            cmd.run(null);
            return;
        }
    } catch (Exception e) {
        jcommander.usage();
        throw e;
    }
    if (arguments.configurationStore == null) {
        System.err.println("Configuration store address argument is required (--configuration-store)");
        jcommander.usage();
        System.exit(1);
    }
    if (arguments.numTransactionCoordinators <= 0) {
        System.err.println("Number of transaction coordinators must greater than 0");
        System.exit(1);
    }
    try (MetadataStoreExtended configStore = PulsarClusterMetadataSetup.initMetadataStore(arguments.configurationStore, arguments.zkSessionTimeoutMillis)) {
        PulsarResources pulsarResources = new PulsarResources(null, configStore);
        // Create system tenant
        PulsarClusterMetadataSetup.createTenantIfAbsent(pulsarResources, NamespaceName.SYSTEM_NAMESPACE.getTenant(), arguments.cluster);
        // Create system namespace
        PulsarClusterMetadataSetup.createNamespaceIfAbsent(pulsarResources, NamespaceName.SYSTEM_NAMESPACE, arguments.cluster);
        // Create transaction coordinator assign partitioned topic
        PulsarClusterMetadataSetup.createPartitionedTopic(configStore, TopicName.TRANSACTION_COORDINATOR_ASSIGN, arguments.numTransactionCoordinators);
    }
    System.out.println("Transaction coordinator metadata setup success");
}

Example 2 with MetadataStoreExtended

use of org.apache.pulsar.metadata.api.extended.MetadataStoreExtended in project pulsar by apache.

the class ClusterMetadataSetupTest method testInitialNamespaceSetup.

@Test
public void testInitialNamespaceSetup() throws Exception {
    // missing arguments
    assertEquals(PulsarInitialNamespaceSetup.doMain(new String[] {}), 1);
    // invalid namespace
    assertEquals(PulsarInitialNamespaceSetup.doMain(new String[] { "--cluster", "testInitialNamespaceSetup-cluster", "--configuration-store", "127.0.0.1:" + localZkS.getZookeeperPort(), "a/b/c/d" }), 1);
    String[] args = { "--cluster", "testInitialNamespaceSetup-cluster", "--configuration-store", "127.0.0.1:" + localZkS.getZookeeperPort(), "test/a", "test/b", "test/c" };
    assertEquals(PulsarInitialNamespaceSetup.doMain(args), 0);
    try (MetadataStoreExtended store = MetadataStoreExtended.create("127.0.0.1:" + localZkS.getZookeeperPort(), MetadataStoreConfig.builder().build())) {
        TenantResources tenantResources = new TenantResources(store, PulsarResources.DEFAULT_OPERATION_TIMEOUT_SEC);
        List<String> namespaces = tenantResources.getListOfNamespaces("test");
        assertEquals(new HashSet<>(namespaces), new HashSet<>(Arrays.asList("test/a", "test/b", "test/c")));
    }
}

Example 3 with MetadataStoreExtended

use of org.apache.pulsar.metadata.api.extended.MetadataStoreExtended in project pulsar by apache.

the class ClusterMetadataSetupTest method testSetupWithBkMetadataServiceUri.

@Test
public void testSetupWithBkMetadataServiceUri() throws Exception {
    String zkConnection = "127.0.0.1:" + localZkS.getZookeeperPort();
    String[] args = { "--cluster", "testReSetupClusterMetadata-cluster", "--zookeeper", zkConnection, "--configuration-store", zkConnection, "--existing-bk-metadata-service-uri", "zk+null://" + zkConnection + "/chroot/ledgers", "--web-service-url", "http://127.0.0.1:8080", "--web-service-url-tls", "https://127.0.0.1:8443", "--broker-service-url", "pulsar://127.0.0.1:6650", "--broker-service-url-tls", "pulsar+ssl://127.0.0.1:6651" };
    PulsarClusterMetadataSetup.main(args);
    try (MetadataStoreExtended localStore = PulsarClusterMetadataSetup.initMetadataStore(zkConnection, 30000)) {
        // expected not exist
        assertFalse(localStore.exists("/ledgers").get());
        String[] bookkeeperMetadataServiceUriArgs = { "--cluster", "testReSetupClusterMetadata-cluster", "--zookeeper", zkConnection, "--configuration-store", zkConnection, "--bookkeeper-metadata-service-uri", "zk+null://" + zkConnection + "/chroot/ledgers", "--web-service-url", "http://127.0.0.1:8080", "--web-service-url-tls", "https://127.0.0.1:8443", "--broker-service-url", "pulsar://127.0.0.1:6650", "--broker-service-url-tls", "pulsar+ssl://127.0.0.1:6651" };
        PulsarClusterMetadataSetup.main(bookkeeperMetadataServiceUriArgs);
        try (MetadataStoreExtended bookkeeperMetadataServiceUriStore = PulsarClusterMetadataSetup.initMetadataStore(zkConnection, 30000)) {
            // expected not exist
            assertFalse(bookkeeperMetadataServiceUriStore.exists("/ledgers").get());
        }
        String[] args1 = { "--cluster", "testReSetupClusterMetadata-cluster", "--zookeeper", zkConnection, "--configuration-store", zkConnection, "--web-service-url", "http://127.0.0.1:8080", "--web-service-url-tls", "https://127.0.0.1:8443", "--broker-service-url", "pulsar://127.0.0.1:6650", "--broker-service-url-tls", "pulsar+ssl://127.0.0.1:6651" };
        PulsarClusterMetadataSetup.main(args1);
        // expected exist
        assertTrue(localStore.exists("/ledgers").get());
    }
}

Example 4 with MetadataStoreExtended

use of org.apache.pulsar.metadata.api.extended.MetadataStoreExtended in project pulsar by apache.

the class NamespaceBundleTest method getNamespaceBundleFactory.

@SuppressWarnings("unchecked")
private NamespaceBundleFactory getNamespaceBundleFactory() {
    PulsarService pulsar = mock(PulsarService.class);
    MetadataStoreExtended store = mock(MetadataStoreExtended.class);
    when(pulsar.getLocalMetadataStore()).thenReturn(store);
    when(pulsar.getConfigurationMetadataStore()).thenReturn(store);
    return NamespaceBundleFactory.createFactory(pulsar, Hashing.crc32());
}

Example 5 with MetadataStoreExtended

use of org.apache.pulsar.metadata.api.extended.MetadataStoreExtended in project pulsar by apache.

the class PulsarLedgerIdGeneratorTest method testGenerateLedgerId.

@Test(dataProvider = "impl")
public void testGenerateLedgerId(String provider, Supplier<String> urlSupplier) throws Exception {
    @Cleanup MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), MetadataStoreConfig.builder().build());
    @Cleanup PulsarLedgerIdGenerator ledgerIdGenerator = new PulsarLedgerIdGenerator(store, "/ledgers");
    // Create *nThread* threads each generate *nLedgers* ledger id,
    // and then check there is no identical ledger id.
    final int nThread = 2;
    final int nLedgers = 2000;
    // Multiply by two. We're going to do half in the old legacy space and half in the new.
    CountDownLatch countDownLatch1 = new CountDownLatch(nThread * nLedgers);
    final AtomicInteger errCount = new AtomicInteger(0);
    final ConcurrentLinkedQueue<Long> ledgerIds = new ConcurrentLinkedQueue<Long>();
    long start = System.currentTimeMillis();
    @Cleanup(value = "shutdownNow") ExecutorService executor = Executors.newCachedThreadPool();
    for (int i = 0; i < nThread; i++) {
        executor.submit(() -> {
            for (int j = 0; j < nLedgers; j++) {
                ledgerIdGenerator.generateLedgerId((rc, result) -> {
                    if (KeeperException.Code.OK.intValue() == rc) {
                        ledgerIds.add(result);
                    } else {
                        errCount.incrementAndGet();
                    }
                    countDownLatch1.countDown();
                });
            }
        });
    }
    countDownLatch1.await();
    CountDownLatch countDownLatch2 = new CountDownLatch(nThread * nLedgers);
    // Go and create the long-id directory in zookeeper. This should cause the id generator to generate ids with the
    // new algo once we clear it's stored status.
    store.put("/ledgers/idgen-long", new byte[0], Optional.empty()).join();
    for (int i = 0; i < nThread; i++) {
        executor.submit(() -> {
            for (int j = 0; j < nLedgers; j++) {
                ledgerIdGenerator.generateLedgerId((rc, result) -> {
                    if (KeeperException.Code.OK.intValue() == rc) {
                        ledgerIds.add(result);
                    } else {
                        errCount.incrementAndGet();
                    }
                    countDownLatch2.countDown();
                });
            }
        });
    }
    assertTrue(countDownLatch2.await(120, TimeUnit.SECONDS), "Wait ledger id generation threads to stop timeout : ");
    log.info("Number of generated ledger id: {}, time used: {}", ledgerIds.size(), System.currentTimeMillis() - start);
    assertEquals(errCount.get(), 0, "Error occur during ledger id generation : ");
    Set<Long> ledgers = new HashSet<>();
    while (!ledgerIds.isEmpty()) {
        Long ledger = ledgerIds.poll();
        assertNotNull(ledger, "Generated ledger id is null");
        assertFalse(ledgers.contains(ledger), "Ledger id [" + ledger + "] conflict : ");
        ledgers.add(ledger);
    }
}