Search in sources :

Example 86 with ZooKeeper

use of org.apache.zookeeper.ZooKeeper in project incubator-pulsar by apache.

the class ZookeeperClientFactoryImplTest method testZKCreationRO.

@Test
void testZKCreationRO() throws Exception {
    ZooKeeperClientFactory zkf = new ZookeeperClientFactoryImpl();
    CompletableFuture<ZooKeeper> zkFuture = zkf.create("127.0.0.1:" + LOCAL_ZOOKEEPER_PORT, SessionType.AllowReadOnly, (int) ZOOKEEPER_SESSION_TIMEOUT_MILLIS);
    localZkc = zkFuture.get(ZOOKEEPER_SESSION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
    assertTrue(localZkc.getState().isConnected());
    localZkc.close();
}
Also used : ZooKeeper(org.apache.zookeeper.ZooKeeper) ZookeeperClientFactoryImpl(org.apache.pulsar.zookeeper.ZookeeperClientFactoryImpl) ZooKeeperClientFactory(org.apache.pulsar.zookeeper.ZooKeeperClientFactory) Test(org.testng.annotations.Test)

Example 87 with ZooKeeper

use of org.apache.zookeeper.ZooKeeper in project incubator-pulsar by apache.

the class PersistentDispatcherFailoverConsumerTest method setup.

@BeforeMethod
public void setup() throws Exception {
    ServiceConfiguration svcConfig = spy(new ServiceConfiguration());
    PulsarService pulsar = spy(new PulsarService(svcConfig));
    doReturn(svcConfig).when(pulsar).getConfiguration();
    mlFactoryMock = mock(ManagedLedgerFactory.class);
    doReturn(mlFactoryMock).when(pulsar).getManagedLedgerFactory();
    ZooKeeper mockZk = createMockZooKeeper();
    doReturn(mockZk).when(pulsar).getZkClient();
    doReturn(createMockBookKeeper(mockZk)).when(pulsar).getBookKeeperClient();
    configCacheService = mock(ConfigurationCacheService.class);
    @SuppressWarnings("unchecked") ZooKeeperDataCache<Policies> zkDataCache = mock(ZooKeeperDataCache.class);
    LocalZooKeeperCacheService zkCache = mock(LocalZooKeeperCacheService.class);
    doReturn(CompletableFuture.completedFuture(Optional.empty())).when(zkDataCache).getAsync(any());
    doReturn(zkDataCache).when(zkCache).policiesCache();
    doReturn(zkDataCache).when(configCacheService).policiesCache();
    doReturn(configCacheService).when(pulsar).getConfigurationCache();
    doReturn(zkCache).when(pulsar).getLocalZkCacheService();
    brokerService = spy(new BrokerService(pulsar));
    doReturn(brokerService).when(pulsar).getBrokerService();
    consumerChanges = new LinkedBlockingQueue<>();
    this.channelCtx = mock(ChannelHandlerContext.class);
    doAnswer(invocationOnMock -> {
        ByteBuf buf = invocationOnMock.getArgumentAt(0, ByteBuf.class);
        ByteBuf cmdBuf = buf.retainedSlice(4, buf.writerIndex() - 4);
        try {
            int cmdSize = (int) cmdBuf.readUnsignedInt();
            int writerIndex = cmdBuf.writerIndex();
            cmdBuf.writerIndex(cmdBuf.readerIndex() + cmdSize);
            ByteBufCodedInputStream cmdInputStream = ByteBufCodedInputStream.get(cmdBuf);
            BaseCommand.Builder cmdBuilder = BaseCommand.newBuilder();
            BaseCommand cmd = cmdBuilder.mergeFrom(cmdInputStream, null).build();
            cmdBuilder.recycle();
            cmdBuf.writerIndex(writerIndex);
            cmdInputStream.recycle();
            if (cmd.hasActiveConsumerChange()) {
                consumerChanges.put(cmd.getActiveConsumerChange());
            }
            cmd.recycle();
        } finally {
            cmdBuf.release();
        }
        return null;
    }).when(channelCtx).writeAndFlush(any(), any());
    serverCnx = spy(new ServerCnx(brokerService));
    doReturn(true).when(serverCnx).isActive();
    doReturn(true).when(serverCnx).isWritable();
    doReturn(new InetSocketAddress("localhost", 1234)).when(serverCnx).clientAddress();
    when(serverCnx.getRemoteEndpointProtocolVersion()).thenReturn(ProtocolVersion.v12.getNumber());
    when(serverCnx.ctx()).thenReturn(channelCtx);
    serverCnxWithOldVersion = spy(new ServerCnx(brokerService));
    doReturn(true).when(serverCnxWithOldVersion).isActive();
    doReturn(true).when(serverCnxWithOldVersion).isWritable();
    doReturn(new InetSocketAddress("localhost", 1234)).when(serverCnxWithOldVersion).clientAddress();
    when(serverCnxWithOldVersion.getRemoteEndpointProtocolVersion()).thenReturn(ProtocolVersion.v11.getNumber());
    when(serverCnxWithOldVersion.ctx()).thenReturn(channelCtx);
    NamespaceService nsSvc = mock(NamespaceService.class);
    doReturn(nsSvc).when(pulsar).getNamespaceService();
    doReturn(true).when(nsSvc).isServiceUnitOwned(any(NamespaceBundle.class));
    doReturn(true).when(nsSvc).isServiceUnitActive(any(TopicName.class));
    setupMLAsyncCallbackMocks();
}
Also used : NamespaceBundle(org.apache.pulsar.common.naming.NamespaceBundle) Policies(org.apache.pulsar.common.policies.data.Policies) BaseCommand(org.apache.pulsar.common.api.proto.PulsarApi.BaseCommand) InetSocketAddress(java.net.InetSocketAddress) ChannelHandlerContext(io.netty.channel.ChannelHandlerContext) ByteBuf(io.netty.buffer.ByteBuf) TopicName(org.apache.pulsar.common.naming.TopicName) ZooKeeper(org.apache.zookeeper.ZooKeeper) MockedPulsarServiceBaseTest.createMockZooKeeper(org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockZooKeeper) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) PulsarService(org.apache.pulsar.broker.PulsarService) NamespaceService(org.apache.pulsar.broker.namespace.NamespaceService) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) ConfigurationCacheService(org.apache.pulsar.broker.cache.ConfigurationCacheService) ByteBufCodedInputStream(org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream) LocalZooKeeperCacheService(org.apache.pulsar.broker.cache.LocalZooKeeperCacheService) BeforeMethod(org.testng.annotations.BeforeMethod)

Example 88 with ZooKeeper

use of org.apache.zookeeper.ZooKeeper in project incubator-pulsar by apache.

the class PersistentTopicTest method setup.

@BeforeMethod
public void setup() throws Exception {
    ServiceConfiguration svcConfig = spy(new ServiceConfiguration());
    pulsar = spy(new PulsarService(svcConfig));
    doReturn(svcConfig).when(pulsar).getConfiguration();
    mlFactoryMock = mock(ManagedLedgerFactory.class);
    doReturn(mlFactoryMock).when(pulsar).getManagedLedgerFactory();
    ZooKeeper mockZk = createMockZooKeeper();
    doReturn(mockZk).when(pulsar).getZkClient();
    doReturn(createMockBookKeeper(mockZk)).when(pulsar).getBookKeeperClient();
    configCacheService = mock(ConfigurationCacheService.class);
    @SuppressWarnings("unchecked") ZooKeeperDataCache<Policies> zkDataCache = mock(ZooKeeperDataCache.class);
    doReturn(zkDataCache).when(configCacheService).policiesCache();
    doReturn(configCacheService).when(pulsar).getConfigurationCache();
    doReturn(Optional.empty()).when(zkDataCache).get(anyString());
    LocalZooKeeperCacheService zkCache = mock(LocalZooKeeperCacheService.class);
    doReturn(CompletableFuture.completedFuture(Optional.empty())).when(zkDataCache).getAsync(any());
    doReturn(zkDataCache).when(zkCache).policiesCache();
    doReturn(configCacheService).when(pulsar).getConfigurationCache();
    doReturn(zkCache).when(pulsar).getLocalZkCacheService();
    brokerService = spy(new BrokerService(pulsar));
    doReturn(brokerService).when(pulsar).getBrokerService();
    serverCnx = spy(new ServerCnx(brokerService));
    doReturn(true).when(serverCnx).isActive();
    doReturn(true).when(serverCnx).isWritable();
    doReturn(new InetSocketAddress("localhost", 1234)).when(serverCnx).clientAddress();
    NamespaceService nsSvc = mock(NamespaceService.class);
    doReturn(nsSvc).when(pulsar).getNamespaceService();
    doReturn(true).when(nsSvc).isServiceUnitOwned(any(NamespaceBundle.class));
    doReturn(true).when(nsSvc).isServiceUnitActive(any(TopicName.class));
    setupMLAsyncCallbackMocks();
}
Also used : NamespaceBundle(org.apache.pulsar.common.naming.NamespaceBundle) Policies(org.apache.pulsar.common.policies.data.Policies) InetSocketAddress(java.net.InetSocketAddress) TopicName(org.apache.pulsar.common.naming.TopicName) ZooKeeper(org.apache.zookeeper.ZooKeeper) MockedPulsarServiceBaseTest.createMockZooKeeper(org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.createMockZooKeeper) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) PulsarService(org.apache.pulsar.broker.PulsarService) NamespaceService(org.apache.pulsar.broker.namespace.NamespaceService) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) ConfigurationCacheService(org.apache.pulsar.broker.cache.ConfigurationCacheService) LocalZooKeeperCacheService(org.apache.pulsar.broker.cache.LocalZooKeeperCacheService) BeforeMethod(org.testng.annotations.BeforeMethod)

Example 89 with ZooKeeper

use of org.apache.zookeeper.ZooKeeper in project incubator-pulsar by apache.

the class ZooKeeperClientAspectJTest method testZkConnected.

@Test
public void testZkConnected() throws Exception {
    OrderedScheduler executor = OrderedScheduler.newSchedulerBuilder().build();
    try {
        ZooKeeperClientFactory zkf = new ZookeeperBkClientFactoryImpl(executor);
        CompletableFuture<ZooKeeper> zkFuture = zkf.create("127.0.0.1:" + LOCAL_ZOOKEEPER_PORT, SessionType.ReadWrite, (int) ZOOKEEPER_SESSION_TIMEOUT_MILLIS);
        localZkc = zkFuture.get(ZOOKEEPER_SESSION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
        assertTrue(localZkc.getState().isConnected());
        assertNotEquals(localZkc.getState(), States.CONNECTEDREADONLY);
    } finally {
        if (localZkc != null) {
            localZkc.close();
        }
        executor.shutdown();
    }
}
Also used : ZooKeeper(org.apache.zookeeper.ZooKeeper) ZookeeperBkClientFactoryImpl(org.apache.pulsar.zookeeper.ZookeeperBkClientFactoryImpl) OrderedScheduler(org.apache.bookkeeper.common.util.OrderedScheduler) ZooKeeperClientFactory(org.apache.pulsar.zookeeper.ZooKeeperClientFactory) Test(org.testng.annotations.Test)

Example 90 with ZooKeeper

use of org.apache.zookeeper.ZooKeeper in project incubator-pulsar by apache.

the class ZooKeeperClientAspectJTest method testZkOpStatsMetrics.

/**
 * Verifies that aspect-advice calculates the latency of of zk-operation and updates PulsarStats
 *
 * @throws Exception
 */
@Test(enabled = false, timeOut = 7000)
public void testZkOpStatsMetrics() throws Exception {
    OrderedScheduler executor = OrderedScheduler.newSchedulerBuilder().build();
    ZooKeeperClientFactory zkf = new ZookeeperBkClientFactoryImpl(executor);
    CompletableFuture<ZooKeeper> zkFuture = zkf.create("127.0.0.1:" + LOCAL_ZOOKEEPER_PORT, SessionType.ReadWrite, (int) ZOOKEEPER_SESSION_TIMEOUT_MILLIS);
    localZkc = zkFuture.get(ZOOKEEPER_SESSION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
    MockPulsar mockPulsar = new MockPulsar(localZkc);
    mockPulsar.setup();
    try {
        PulsarClient pulsarClient = mockPulsar.getClient();
        PulsarService pulsar = mockPulsar.getPulsar();
        pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
        Metrics zkOpMetric = getMetric(pulsar, "zk_write_latency");
        Assert.assertNotNull(zkOpMetric);
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_rate_s"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_95percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_99_99_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_99_9_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_99_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_mean_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_time_median_ms"));
        zkOpMetric = getMetric(pulsar, "zk_read_latency");
        Assert.assertNotNull(zkOpMetric);
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_rate_s"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_95percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_99_99_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_99_9_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_99_percentile_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_mean_ms"));
        Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_read_time_median_ms"));
        CountDownLatch createLatch = new CountDownLatch(1);
        CountDownLatch deleteLatch = new CountDownLatch(1);
        CountDownLatch readLatch = new CountDownLatch(1);
        CountDownLatch existLatch = new CountDownLatch(1);
        localZkc.create("/createTest", "data".getBytes(), Acl, CreateMode.EPHEMERAL, (rc, path, ctx, name) -> {
            createLatch.countDown();
        }, "create");
        localZkc.delete("/deleteTest", -1, (rc, path, ctx) -> {
            deleteLatch.countDown();
        }, "delete");
        localZkc.exists("/createTest", null, (int rc, String path, Object ctx, Stat stat) -> {
            existLatch.countDown();
        }, null);
        localZkc.getData("/createTest", null, (int rc, String path, Object ctx, byte[] data, Stat stat) -> {
            readLatch.countDown();
        }, null);
        createLatch.await();
        deleteLatch.await();
        existLatch.await();
        readLatch.await();
        Thread.sleep(10);
        BrokerService brokerService = pulsar.getBrokerService();
        brokerService.updateRates();
        List<Metrics> metrics = brokerService.getTopicMetrics();
        AtomicDouble writeRate = new AtomicDouble();
        AtomicDouble readRate = new AtomicDouble();
        metrics.forEach(m -> {
            if ("zk_write_latency".equalsIgnoreCase(m.getDimension("metric"))) {
                writeRate.set((double) m.getMetrics().get("brk_zk_write_latency_rate_s"));
            } else if ("zk_read_latency".equalsIgnoreCase(m.getDimension("metric"))) {
                readRate.set((double) m.getMetrics().get("brk_zk_read_latency_rate_s"));
            }
        });
        Assert.assertTrue(readRate.get() > 0);
        Assert.assertTrue(writeRate.get() > 0);
    } finally {
        mockPulsar.cleanup();
        if (localZkc != null) {
            localZkc.close();
        }
        executor.shutdown();
    }
}
Also used : AtomicDouble(com.google.common.util.concurrent.AtomicDouble) CountDownLatch(java.util.concurrent.CountDownLatch) ZooKeeperClientFactory(org.apache.pulsar.zookeeper.ZooKeeperClientFactory) Metrics(org.apache.pulsar.common.stats.Metrics) ZooKeeper(org.apache.zookeeper.ZooKeeper) Stat(org.apache.zookeeper.data.Stat) PulsarService(org.apache.pulsar.broker.PulsarService) ZookeeperBkClientFactoryImpl(org.apache.pulsar.zookeeper.ZookeeperBkClientFactoryImpl) PulsarClient(org.apache.pulsar.client.api.PulsarClient) BrokerService(org.apache.pulsar.broker.service.BrokerService) OrderedScheduler(org.apache.bookkeeper.common.util.OrderedScheduler) Test(org.testng.annotations.Test)

Aggregations

ZooKeeper (org.apache.zookeeper.ZooKeeper)605 Test (org.junit.jupiter.api.Test)190 KeeperException (org.apache.zookeeper.KeeperException)153 Test (org.junit.Test)100 Stat (org.apache.zookeeper.data.Stat)86 IOException (java.io.IOException)83 CountDownLatch (java.util.concurrent.CountDownLatch)80 WatchedEvent (org.apache.zookeeper.WatchedEvent)70 Watcher (org.apache.zookeeper.Watcher)59 ArrayList (java.util.ArrayList)47 CountdownWatcher (org.apache.zookeeper.test.ClientBase.CountdownWatcher)41 TestableZooKeeper (org.apache.zookeeper.TestableZooKeeper)38 Timeout (org.junit.jupiter.api.Timeout)32 ZooKeeperServer (org.apache.zookeeper.server.ZooKeeperServer)31 File (java.io.File)28 HashMap (java.util.HashMap)26 CompletableFuture (java.util.concurrent.CompletableFuture)22 Test (org.testng.annotations.Test)21 AsyncCallback (org.apache.zookeeper.AsyncCallback)19 ACL (org.apache.zookeeper.data.ACL)19