use of com.yahoo.pulsar.zookeeper.LocalBookkeeperEnsemble in project pulsar by yahoo.
the class SLAMonitoringTest method setup.
@BeforeClass
void setup() throws Exception {
log.info("---- Initializing SLAMonitoringTest -----");
// Start local bookkeeper ensemble
bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, PortManager.nextFreePort());
bkEnsemble.start();
// start brokers
for (int i = 0; i < BROKER_COUNT; i++) {
brokerWebServicePorts[i] = PortManager.nextFreePort();
brokerNativeBrokerPorts[i] = PortManager.nextFreePort();
ServiceConfiguration config = new ServiceConfiguration();
config.setBrokerServicePort(brokerNativeBrokerPorts[i]);
config.setClusterName("my-cluster");
config.setWebServicePort(brokerWebServicePorts[i]);
config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
config.setBrokerServicePort(brokerNativeBrokerPorts[i]);
configurations[i] = config;
pulsarServices[i] = new PulsarService(config);
pulsarServices[i].start();
brokerUrls[i] = new URL("http://127.0.0.1" + ":" + brokerWebServicePorts[i]);
pulsarAdmins[i] = new PulsarAdmin(brokerUrls[i], (Authentication) null);
}
Thread.sleep(100);
createProperty(pulsarAdmins[BROKER_COUNT - 1]);
for (int i = 0; i < BROKER_COUNT; i++) {
String destination = String.format("%s/%s/%s:%s", NamespaceService.SLA_NAMESPACE_PROPERTY, "my-cluster", pulsarServices[i].getAdvertisedAddress(), brokerWebServicePorts[i]);
pulsarAdmins[0].namespaces().createNamespace(destination);
}
}
use of com.yahoo.pulsar.zookeeper.LocalBookkeeperEnsemble in project pulsar by yahoo.
the class LoadBalancerTest method setup.
@BeforeMethod
void setup() throws Exception {
// Start local bookkeeper ensemble
bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, PortManager.nextFreePort());
bkEnsemble.start();
ZkUtils.createFullPathOptimistic(bkEnsemble.getZkClient(), SimpleLoadManagerImpl.LOADBALANCER_DYNAMIC_SETTING_STRATEGY_ZPATH, "{\"loadBalancerStrategy\":\"leastLoadedServer\"}".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
// start brokers
for (int i = 0; i < BROKER_COUNT; i++) {
brokerWebServicePorts[i] = PortManager.nextFreePort();
brokerNativeBrokerPorts[i] = PortManager.nextFreePort();
ServiceConfiguration config = new ServiceConfiguration();
config.setBrokerServicePort(brokerNativeBrokerPorts[i]);
config.setClusterName("use");
config.setWebServicePort(brokerWebServicePorts[i]);
config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
config.setBrokerServicePort(brokerNativeBrokerPorts[i]);
pulsarServices[i] = new PulsarService(config);
pulsarServices[i].start();
brokerUrls[i] = new URL("http://127.0.0.1" + ":" + brokerWebServicePorts[i]);
lookupAddresses[i] = pulsarServices[i].getAdvertisedAddress() + ":" + config.getWebServicePort();
pulsarAdmins[i] = new PulsarAdmin(brokerUrls[i], (Authentication) null);
}
createNamespacePolicies(pulsarServices[0]);
Thread.sleep(100);
}
use of com.yahoo.pulsar.zookeeper.LocalBookkeeperEnsemble in project pulsar by yahoo.
the class BacklogQuotaManagerTest method setup.
@BeforeMethod
void setup() throws Exception {
try {
// start local bookie and zookeeper
bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, 5001);
bkEnsemble.start();
// start pulsar service
config = new ServiceConfiguration();
config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
config.setWebServicePort(BROKER_WEBSERVICE_PORT);
config.setClusterName("usc");
config.setBrokerServicePort(BROKER_SERVICE_PORT);
config.setAuthorizationEnabled(false);
config.setAuthenticationEnabled(false);
config.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA);
config.setManagedLedgerMaxEntriesPerLedger(5);
config.setManagedLedgerMinLedgerRolloverTimeMinutes(0);
pulsar = new PulsarService(config);
pulsar.start();
adminUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
admin = new PulsarAdmin(adminUrl, (Authentication) null);
admin.clusters().createCluster("usc", new ClusterData(adminUrl.toString()));
admin.properties().createProperty("prop", new PropertyAdmin(Lists.newArrayList("appid1"), Sets.newHashSet("usc")));
admin.namespaces().createNamespace("prop/usc/ns-quota");
admin.namespaces().createNamespace("prop/usc/quotahold");
admin.namespaces().createNamespace("prop/usc/quotaholdasync");
} catch (Throwable t) {
LOG.error("Error setting up broker test", t);
Assert.fail("Broker test setup failed");
}
}
use of com.yahoo.pulsar.zookeeper.LocalBookkeeperEnsemble in project pulsar by yahoo.
the class SimpleLoadManagerImplTest method setup.
@BeforeMethod
void setup() throws Exception {
// Start local bookkeeper ensemble
bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, PortManager.nextFreePort());
bkEnsemble.start();
// Start broker 1
ServiceConfiguration config1 = spy(new ServiceConfiguration());
config1.setClusterName("use");
config1.setWebServicePort(PRIMARY_BROKER_WEBSERVICE_PORT);
config1.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
config1.setBrokerServicePort(PRIMARY_BROKER_PORT);
pulsar1 = new PulsarService(config1);
pulsar1.start();
url1 = new URL("http://127.0.0.1" + ":" + PRIMARY_BROKER_WEBSERVICE_PORT);
admin1 = new PulsarAdmin(url1, (Authentication) null);
brokerStatsClient1 = admin1.brokerStats();
// Start broker 2
ServiceConfiguration config2 = new ServiceConfiguration();
config2.setClusterName("use");
config2.setWebServicePort(SECONDARY_BROKER_WEBSERVICE_PORT);
config2.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
config2.setBrokerServicePort(SECONDARY_BROKER_PORT);
pulsar2 = new PulsarService(config2);
pulsar2.start();
url2 = new URL("http://127.0.0.1" + ":" + SECONDARY_BROKER_WEBSERVICE_PORT);
admin2 = new PulsarAdmin(url2, (Authentication) null);
brokerStatsClient2 = admin2.brokerStats();
createNamespacePolicies(pulsar1);
Thread.sleep(100);
}
use of com.yahoo.pulsar.zookeeper.LocalBookkeeperEnsemble in project pulsar by yahoo.
the class ReplicatorTestBase method setup.
void setup() throws Exception {
log.info("--- Starting ReplicatorTestBase::setup ---");
int globalZKPort = PortManager.nextFreePort();
globalZkS = new ZookeeperServerTest(globalZKPort);
globalZkS.start();
// Start region 1
int zkPort1 = PortManager.nextFreePort();
bkEnsemble1 = new LocalBookkeeperEnsemble(3, zkPort1, PortManager.nextFreePort());
bkEnsemble1.start();
int webServicePort1 = PortManager.nextFreePort();
// NOTE: we have to instantiate a new copy of System.getProperties() to make sure pulsar1 and pulsar2 have
// completely
// independent config objects instead of referring to the same properties object
ServiceConfiguration config1 = new ServiceConfiguration();
config1.setClusterName("r1");
config1.setWebServicePort(webServicePort1);
config1.setZookeeperServers("127.0.0.1:" + zkPort1);
config1.setGlobalZookeeperServers("127.0.0.1:" + globalZKPort + "/foo");
config1.setBrokerDeleteInactiveTopicsEnabled(isBrokerServicePurgeInactiveDestination());
config1.setBrokerServicePurgeInactiveFrequencyInSeconds(inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS));
config1.setBrokerServicePort(PortManager.nextFreePort());
config1.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA);
pulsar1 = new PulsarService(config1);
pulsar1.start();
ns1 = pulsar1.getBrokerService();
url1 = new URL("http://127.0.0.1:" + webServicePort1);
admin1 = new PulsarAdmin(url1, (Authentication) null);
// Start region 2
// Start zk & bks
int zkPort2 = PortManager.nextFreePort();
bkEnsemble2 = new LocalBookkeeperEnsemble(3, zkPort2, PortManager.nextFreePort());
bkEnsemble2.start();
int webServicePort2 = PortManager.nextFreePort();
config2 = new ServiceConfiguration();
config2.setClusterName("r2");
config2.setWebServicePort(webServicePort2);
config2.setZookeeperServers("127.0.0.1:" + zkPort2);
config2.setGlobalZookeeperServers("127.0.0.1:" + globalZKPort + "/foo");
config2.setBrokerDeleteInactiveTopicsEnabled(isBrokerServicePurgeInactiveDestination());
config2.setBrokerServicePurgeInactiveFrequencyInSeconds(inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS));
config2.setBrokerServicePort(PortManager.nextFreePort());
config2.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA);
pulsar2 = new PulsarService(config2);
pulsar2.start();
ns2 = pulsar2.getBrokerService();
url2 = new URL("http://127.0.0.1:" + webServicePort2);
admin2 = new PulsarAdmin(url2, (Authentication) null);
// Start region 3
// Start zk & bks
int zkPort3 = PortManager.nextFreePort();
bkEnsemble3 = new LocalBookkeeperEnsemble(3, zkPort3, PortManager.nextFreePort());
bkEnsemble3.start();
int webServicePort3 = PortManager.nextFreePort();
config3 = new ServiceConfiguration();
config3.setClusterName("r3");
config3.setWebServicePort(webServicePort3);
config3.setZookeeperServers("127.0.0.1:" + zkPort3);
config3.setGlobalZookeeperServers("127.0.0.1:" + globalZKPort + "/foo");
config3.setBrokerDeleteInactiveTopicsEnabled(isBrokerServicePurgeInactiveDestination());
config3.setBrokerServicePurgeInactiveFrequencyInSeconds(inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS));
config3.setBrokerServicePort(PortManager.nextFreePort());
pulsar3 = new PulsarService(config3);
pulsar3.start();
ns3 = pulsar3.getBrokerService();
url3 = new URL("http://127.0.0.1:" + webServicePort3);
admin3 = new PulsarAdmin(url3, (Authentication) null);
// Provision the global namespace
admin1.clusters().createCluster("r1", new ClusterData(url1.toString(), null, pulsar1.getBrokerServiceUrl(), pulsar1.getBrokerServiceUrlTls()));
admin1.clusters().createCluster("r2", new ClusterData(url2.toString(), null, pulsar2.getBrokerServiceUrl(), pulsar1.getBrokerServiceUrlTls()));
admin1.clusters().createCluster("r3", new ClusterData(url3.toString(), null, pulsar3.getBrokerServiceUrl(), pulsar1.getBrokerServiceUrlTls()));
admin1.clusters().createCluster("global", new ClusterData("http://global:8080"));
admin1.properties().createProperty("pulsar", new PropertyAdmin(Lists.newArrayList("appid1", "appid2", "appid3"), Sets.newHashSet("r1", "r2", "r3")));
admin1.namespaces().createNamespace("pulsar/global/ns");
admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Lists.newArrayList("r1", "r2", "r3"));
admin1.namespaces().createNamespace("pulsar/global/ns1");
admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns1", Lists.newArrayList("r1", "r2"));
assertEquals(admin2.clusters().getCluster("r1").getServiceUrl(), url1.toString());
assertEquals(admin2.clusters().getCluster("r2").getServiceUrl(), url2.toString());
assertEquals(admin2.clusters().getCluster("r3").getServiceUrl(), url3.toString());
assertEquals(admin2.clusters().getCluster("r1").getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl());
assertEquals(admin2.clusters().getCluster("r2").getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl());
assertEquals(admin2.clusters().getCluster("r3").getBrokerServiceUrl(), pulsar3.getBrokerServiceUrl());
/*
* assertEquals(admin2.clusters().getCluster("global").getServiceUrl(), "http://global:8080");
* assertEquals(admin2.properties().getPropertyAdmin("pulsar").getAdminRoles(), Lists.newArrayList("appid1",
* "appid2")); assertEquals(admin2.namespaces().getPolicies("pulsar/global/ns").replication_clusters,
* Lists.newArrayList("r1", "r2", "r3"));
*
* admin1.namespaces().createNamespace("pulsar/global/ns2");
* admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns2", Lists.newArrayList("r1", "r2",
* "r3"));
*/
Thread.sleep(100);
log.info("--- ReplicatorTestBase::setup completed ---");
}
Aggregations