use of org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble in project incubator-pulsar by apache.
the class PulsarStandaloneStarter method start.
void start() throws Exception {
if (config == null) {
System.exit(1);
}
log.debug("--- setup PulsarStandaloneStarter ---");
if (!onlyBroker) {
// Start LocalBookKeeper
bkEnsemble = new LocalBookkeeperEnsemble(numOfBk, zkPort, bkPort, zkDir, bkDir, wipeData, config.getAdvertisedAddress());
bkEnsemble.startStandalone();
}
if (noBroker) {
return;
}
// load aspectj-weaver agent for instrumentation
AgentLoader.loadAgentClass(Agent.class.getName(), null);
// initialize the functions worker
if (!noFunctionsWorker) {
WorkerConfig workerConfig;
if (isBlank(fnWorkerConfigFile)) {
workerConfig = new WorkerConfig();
} else {
workerConfig = WorkerConfig.load(fnWorkerConfigFile);
}
// worker talks to local broker
workerConfig.setPulsarServiceUrl("pulsar://127.0.0.1:" + config.getBrokerServicePort());
workerConfig.setPulsarWebServiceUrl("http://127.0.0.1:" + config.getWebServicePort());
String hostname = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(config.getAdvertisedAddress());
workerConfig.setWorkerHostname(hostname);
workerConfig.setWorkerId("c-" + config.getClusterName() + "-fw-" + hostname + "-" + workerConfig.getWorkerPort());
fnWorkerService = new WorkerService(workerConfig);
}
// Start Broker
broker = new PulsarService(config, Optional.ofNullable(fnWorkerService));
broker.start();
// Create a sample namespace
URL webServiceUrl = new URL(String.format("http://%s:%d", config.getAdvertisedAddress(), config.getWebServicePort()));
final String brokerServiceUrl = String.format("pulsar://%s:%d", config.getAdvertisedAddress(), config.getBrokerServicePort());
admin = new PulsarAdmin(webServiceUrl, config.getBrokerClientAuthenticationPlugin(), config.getBrokerClientAuthenticationParameters());
final String property = "sample";
final String cluster = config.getClusterName();
final String globalCluster = "global";
final String namespace = property + "/" + cluster + "/ns1";
try {
ClusterData clusterData = new ClusterData(webServiceUrl.toString(), null, /* serviceUrlTls */
brokerServiceUrl, null);
if (!admin.clusters().getClusters().contains(cluster)) {
admin.clusters().createCluster(cluster, clusterData);
} else {
admin.clusters().updateCluster(cluster, clusterData);
}
// Create marker for "global" cluster
if (!admin.clusters().getClusters().contains(globalCluster)) {
admin.clusters().createCluster(globalCluster, new ClusterData(null, null));
}
if (!admin.properties().getProperties().contains(property)) {
admin.properties().createProperty(property, new PropertyAdmin(Lists.newArrayList(config.getSuperUserRoles()), Sets.newHashSet(cluster)));
}
if (!admin.namespaces().getNamespaces(property).contains(namespace)) {
admin.namespaces().createNamespace(namespace);
}
} catch (PulsarAdminException e) {
log.info(e.getMessage());
}
if (null != fnWorkerService) {
fnWorkerService.start();
}
log.debug("--- setup completed ---");
}
use of org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble in project incubator-pulsar by apache.
the class LocalBookkeeperEnsembleTest method testDataDirectoryCreatingAndRemoving.
@Test
void testDataDirectoryCreatingAndRemoving() throws Exception {
final int numBk = 1;
final int zkPort = PortManager.nextFreePort();
final int bkPort = PortManager.nextFreePort();
final String zkDirName = "/tmp/data/zookeeper";
File zkDir = new File(zkDirName);
final String bkDirName = "/tmp/data/bookkeeper";
File bkDir = new File(bkDirName + "0");
// At first delete existing data directories
FileUtils.deleteDirectory(zkDir);
FileUtils.deleteDirectory(bkDir);
assertFalse(zkDir.exists());
assertFalse(bkDir.exists());
// Start local Bookies/ZooKeepers and confirm that specified data directories are created
LocalBookkeeperEnsemble ensemble1 = new LocalBookkeeperEnsemble(numBk, zkPort, bkPort, zkDirName, bkDirName, true);
ensemble1.start();
assertTrue(zkDir.exists());
assertTrue(bkDir.exists());
ensemble1.stop();
// Restart local Bookies/ZooKeepers without refreshing data
LocalBookkeeperEnsemble ensemble2 = new LocalBookkeeperEnsemble(numBk, zkPort, bkPort, zkDirName, bkDirName, false);
ensemble2.start();
assertTrue(ensemble2.getZkServer().isRunning());
assertEquals(ensemble2.getZkServer().getClientPort(), zkPort);
assertTrue(ensemble2.getZkClient().getState().isConnected());
assertTrue(ensemble2.getBookies()[0].isRunning());
assertEquals(ensemble2.getBookies()[0].getLocalAddress().getPort(), bkPort);
// Stop local Bookies/ZooKeepers and confirm that they are correctly closed
ensemble2.stop();
assertFalse(ensemble2.getZkServer().isRunning());
assertFalse(ensemble2.getZkClient().getState().isConnected());
assertFalse(ensemble2.getBookies()[0].isRunning());
// Finaly delete data directories
FileUtils.deleteDirectory(zkDir);
FileUtils.deleteDirectory(bkDir);
}
use of org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble in project incubator-pulsar by apache.
the class LocalBookkeeperEnsembleTest method testAdvertisedAddress.
@Test
void testAdvertisedAddress() throws Exception {
final int numBk = 1;
final int zkPort = PortManager.nextFreePort();
final int bkPort = PortManager.nextFreePort();
LocalBookkeeperEnsemble ensemble = new LocalBookkeeperEnsemble(numBk, zkPort, bkPort, null, null, true, "127.0.0.2");
ensemble.startStandalone();
assertNotNull(ensemble.getZkClient().exists("/ledgers/available/127.0.0.2:" + bkPort, false));
ensemble.stop();
}
use of org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble in project incubator-pulsar by apache.
the class ReplicatorTestBase method setup.
void setup() throws Exception {
log.info("--- Starting ReplicatorTestBase::setup ---");
int globalZKPort = PortManager.nextFreePort();
globalZkS = new ZookeeperServerTest(globalZKPort);
globalZkS.start();
// Start region 1
int zkPort1 = PortManager.nextFreePort();
bkEnsemble1 = new LocalBookkeeperEnsemble(3, zkPort1, PortManager.nextFreePort());
bkEnsemble1.start();
int webServicePort1 = PortManager.nextFreePort();
int webServicePortTls1 = PortManager.nextFreePort();
// NOTE: we have to instantiate a new copy of System.getProperties() to make sure pulsar1 and pulsar2 have
// completely
// independent config objects instead of referring to the same properties object
config1.setClusterName("r1");
config1.setAdvertisedAddress("localhost");
config1.setWebServicePort(webServicePort1);
config1.setWebServicePortTls(webServicePortTls1);
config1.setZookeeperServers("127.0.0.1:" + zkPort1);
config1.setGlobalZookeeperServers("127.0.0.1:" + globalZKPort + "/foo");
config1.setBrokerDeleteInactiveTopicsEnabled(isBrokerServicePurgeInactiveTopic());
config1.setBrokerServicePurgeInactiveFrequencyInSeconds(inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS));
config1.setBrokerServicePort(PortManager.nextFreePort());
config1.setBrokerServicePortTls(PortManager.nextFreePort());
config1.setTlsEnabled(true);
config1.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
config1.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
config1.setTlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH);
config1.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA);
config1.setDefaultNumberOfNamespaceBundles(1);
pulsar1 = new PulsarService(config1);
pulsar1.start();
ns1 = pulsar1.getBrokerService();
url1 = new URL("http://localhost:" + webServicePort1);
urlTls1 = new URL("https://localhost:" + webServicePortTls1);
admin1 = new PulsarAdmin(url1, (Authentication) null);
// Start region 2
// Start zk & bks
int zkPort2 = PortManager.nextFreePort();
bkEnsemble2 = new LocalBookkeeperEnsemble(3, zkPort2, PortManager.nextFreePort());
bkEnsemble2.start();
int webServicePort2 = PortManager.nextFreePort();
int webServicePortTls2 = PortManager.nextFreePort();
config2.setClusterName("r2");
config2.setAdvertisedAddress("localhost");
config2.setWebServicePort(webServicePort2);
config2.setWebServicePortTls(webServicePortTls2);
config2.setZookeeperServers("127.0.0.1:" + zkPort2);
config2.setGlobalZookeeperServers("127.0.0.1:" + globalZKPort + "/foo");
config2.setBrokerDeleteInactiveTopicsEnabled(isBrokerServicePurgeInactiveTopic());
config2.setBrokerServicePurgeInactiveFrequencyInSeconds(inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS));
config2.setBrokerServicePort(PortManager.nextFreePort());
config2.setBrokerServicePortTls(PortManager.nextFreePort());
config2.setTlsEnabled(true);
config2.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
config2.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
config2.setTlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH);
config2.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA);
config2.setDefaultNumberOfNamespaceBundles(1);
pulsar2 = new PulsarService(config2);
pulsar2.start();
ns2 = pulsar2.getBrokerService();
url2 = new URL("http://localhost:" + webServicePort2);
urlTls2 = new URL("https://localhost:" + webServicePortTls2);
admin2 = new PulsarAdmin(url2, (Authentication) null);
// Start region 3
// Start zk & bks
int zkPort3 = PortManager.nextFreePort();
bkEnsemble3 = new LocalBookkeeperEnsemble(3, zkPort3, PortManager.nextFreePort());
bkEnsemble3.start();
int webServicePort3 = PortManager.nextFreePort();
int webServicePortTls3 = PortManager.nextFreePort();
config3.setClusterName("r3");
config3.setAdvertisedAddress("localhost");
config3.setWebServicePort(webServicePort3);
config3.setWebServicePortTls(webServicePortTls3);
config3.setZookeeperServers("127.0.0.1:" + zkPort3);
config3.setGlobalZookeeperServers("127.0.0.1:" + globalZKPort + "/foo");
config3.setBrokerDeleteInactiveTopicsEnabled(isBrokerServicePurgeInactiveTopic());
config3.setBrokerServicePurgeInactiveFrequencyInSeconds(inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS));
config3.setBrokerServicePort(PortManager.nextFreePort());
config3.setBrokerServicePortTls(PortManager.nextFreePort());
config3.setTlsEnabled(true);
config3.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
config3.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
config3.setTlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH);
config3.setDefaultNumberOfNamespaceBundles(1);
pulsar3 = new PulsarService(config3);
pulsar3.start();
ns3 = pulsar3.getBrokerService();
url3 = new URL("http://localhost:" + webServicePort3);
urlTls3 = new URL("https://localhost:" + webServicePortTls3);
admin3 = new PulsarAdmin(url3, (Authentication) null);
// Provision the global namespace
admin1.clusters().createCluster("r1", new ClusterData(url1.toString(), urlTls1.toString(), pulsar1.getBrokerServiceUrl(), pulsar1.getBrokerServiceUrlTls()));
admin1.clusters().createCluster("r2", new ClusterData(url2.toString(), urlTls2.toString(), pulsar2.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrlTls()));
admin1.clusters().createCluster("r3", new ClusterData(url3.toString(), urlTls3.toString(), pulsar3.getBrokerServiceUrl(), pulsar3.getBrokerServiceUrlTls()));
admin1.clusters().createCluster("global", new ClusterData("http://global:8080", "https://global:8443"));
admin1.properties().createProperty("pulsar", new PropertyAdmin(Lists.newArrayList("appid1", "appid2", "appid3"), Sets.newHashSet("r1", "r2", "r3")));
admin1.namespaces().createNamespace("pulsar/global/ns");
admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Lists.newArrayList("r1", "r2", "r3"));
admin1.namespaces().createNamespace("pulsar/global/ns1");
admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns1", Lists.newArrayList("r1", "r2"));
assertEquals(admin2.clusters().getCluster("r1").getServiceUrl(), url1.toString());
assertEquals(admin2.clusters().getCluster("r2").getServiceUrl(), url2.toString());
assertEquals(admin2.clusters().getCluster("r3").getServiceUrl(), url3.toString());
assertEquals(admin2.clusters().getCluster("r1").getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl());
assertEquals(admin2.clusters().getCluster("r2").getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl());
assertEquals(admin2.clusters().getCluster("r3").getBrokerServiceUrl(), pulsar3.getBrokerServiceUrl());
Thread.sleep(100);
log.info("--- ReplicatorTestBase::setup completed ---");
}
use of org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble in project incubator-pulsar by apache.
the class LoadBalancerTest method setup.
@BeforeMethod
void setup() throws Exception {
// Start local bookkeeper ensemble
bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, PortManager.nextFreePort());
bkEnsemble.start();
ZkUtils.createFullPathOptimistic(bkEnsemble.getZkClient(), SimpleLoadManagerImpl.LOADBALANCER_DYNAMIC_SETTING_STRATEGY_ZPATH, "{\"loadBalancerStrategy\":\"leastLoadedServer\"}".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
final String localhost = InetAddress.getLocalHost().getHostName();
// start brokers
for (int i = 0; i < BROKER_COUNT; i++) {
brokerWebServicePorts[i] = PortManager.nextFreePort();
brokerNativeBrokerPorts[i] = PortManager.nextFreePort();
ServiceConfiguration config = new ServiceConfiguration();
config.setBrokerServicePort(brokerNativeBrokerPorts[i]);
config.setClusterName("use");
config.setAdvertisedAddress("localhost");
config.setWebServicePort(brokerWebServicePorts[i]);
config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
config.setBrokerServicePort(brokerNativeBrokerPorts[i]);
config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
config.setAdvertisedAddress(localhost + i);
config.setLoadBalancerEnabled(false);
pulsarServices[i] = new PulsarService(config);
pulsarServices[i].start();
brokerUrls[i] = new URL("http://127.0.0.1" + ":" + brokerWebServicePorts[i]);
lookupAddresses[i] = pulsarServices[i].getAdvertisedAddress() + ":" + config.getWebServicePort();
pulsarAdmins[i] = new PulsarAdmin(brokerUrls[i], (Authentication) null);
}
createNamespacePolicies(pulsarServices[0]);
Thread.sleep(100);
}
Aggregations