use of org.apache.metron.maas.discovery.ServiceDiscoverer in project metron by apache.
the class MaaSHandler method start.
public void start() throws Exception {
if (client == null) {
RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
client = CuratorFrameworkFactory.newClient(zkQuorum, retryPolicy);
client.start();
}
config = ConfigUtil.INSTANCE.read(client, root, new MaaSConfig(), MaaSConfig.class);
cache = new NodeCache(client, root);
cache.getListenable().addListener(() -> {
byte[] data = cache.getCurrentData().getData();
Lock wLock = lock.writeLock();
wLock.lock();
try {
config = _mapper.readValue(data, MaaSConfig.class);
} finally {
wLock.unlock();
}
});
discoverer = new ServiceDiscoverer(client, config.getServiceRoot());
discoverer.start();
}
use of org.apache.metron.maas.discovery.ServiceDiscoverer in project metron by apache.
the class ModelSubmission method execute.
public void execute(FileSystem fs, String... argv) throws Exception {
CommandLine cli = ModelSubmissionOptions.parse(new PosixParser(), argv);
if (ModelSubmissionOptions.LOG4J_PROPERTIES.has(cli)) {
Log4jPropertyHelper.updateLog4jConfiguration(ModelSubmission.class, ModelSubmissionOptions.LOG4J_PROPERTIES.get(cli));
}
ModelRequest request = null;
CuratorFramework client = null;
try {
RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
client = CuratorFrameworkFactory.newClient(ModelSubmissionOptions.ZK_QUORUM.get(cli), retryPolicy);
client.start();
MaaSConfig config = ConfigUtil.INSTANCE.read(client, ModelSubmissionOptions.ZK_ROOT.get(cli, "/metron/maas/config"), new MaaSConfig(), MaaSConfig.class);
String mode = ModelSubmissionOptions.MODE.get(cli);
if (mode.equalsIgnoreCase("ADD")) {
request = new ModelRequest() {
{
setName(ModelSubmissionOptions.NAME.get(cli));
setAction(Action.ADD);
setVersion(ModelSubmissionOptions.VERSION.get(cli));
setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli)));
setMemory(Integer.parseInt(ModelSubmissionOptions.MEMORY.get(cli)));
setPath(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli));
}
};
} else if (mode.equalsIgnoreCase("REMOVE")) {
request = new ModelRequest() {
{
setName(ModelSubmissionOptions.NAME.get(cli));
setAction(Action.REMOVE);
setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli)));
setVersion(ModelSubmissionOptions.VERSION.get(cli));
}
};
} else if (mode.equalsIgnoreCase("LIST")) {
String name = ModelSubmissionOptions.NAME.get(cli, null);
String version = ModelSubmissionOptions.VERSION.get(cli, null);
ServiceDiscoverer serviceDiscoverer = new ServiceDiscoverer(client, config.getServiceRoot());
Model model = new Model(name, version);
Map<Model, List<ModelEndpoint>> endpoints = serviceDiscoverer.listEndpoints(model);
for (Map.Entry<Model, List<ModelEndpoint>> kv : endpoints.entrySet()) {
String modelTitle = "Model " + kv.getKey().getName() + " @ " + kv.getKey().getVersion();
System.out.println(modelTitle);
for (ModelEndpoint endpoint : kv.getValue()) {
System.out.println(endpoint);
}
}
}
if (ModelSubmissionOptions.LOCAL_MODEL_PATH.has(cli)) {
File localDir = new File(ModelSubmissionOptions.LOCAL_MODEL_PATH.get(cli));
Path hdfsPath = new Path(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli));
updateHDFS(fs, localDir, hdfsPath);
}
Queue<ModelRequest> queue = config.createQueue(ImmutableMap.of(ZKQueue.ZK_CLIENT, client));
queue.enqueue(request);
} finally {
if (client != null) {
client.close();
}
}
}
use of org.apache.metron.maas.discovery.ServiceDiscoverer in project metron by apache.
the class MaasIntegrationTest method testDSShell.
public void testDSShell(boolean haveDomain) throws Exception {
MaaSConfig config = new MaaSConfig() {
{
setServiceRoot("/maas/service");
setQueueConfig(new HashMap<String, Object>() {
{
put(ZKQueue.ZK_PATH, "/maas/queue");
}
});
}
};
String configRoot = "/maas/config";
byte[] configData = ConfigUtil.INSTANCE.toBytes(config);
try {
client.setData().forPath(configRoot, configData);
} catch (KeeperException.NoNodeException e) {
client.create().creatingParentsIfNeeded().forPath(configRoot, configData);
}
String[] args = { "--jar", yarnComponent.getAppMasterJar(), "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--master_memory", "512", "--master_vcores", "2" };
if (haveDomain) {
String[] domainArgs = { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group", "--modify_acls", "writer_user writer_group", "--create" };
List<String> argsList = new ArrayList<String>(Arrays.asList(args));
argsList.addAll(Arrays.asList(domainArgs));
args = argsList.toArray(new String[argsList.size()]);
}
YarnConfiguration conf = yarnComponent.getConfig();
LOG.info("Initializing DS Client");
final Client client = new Client(new Configuration(conf));
boolean initSuccess = client.init(args);
Assert.assertTrue(initSuccess);
LOG.info("Running DS Client");
final AtomicBoolean result = new AtomicBoolean(false);
Thread t = new Thread() {
@Override
public void run() {
try {
result.set(client.run());
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
t.start();
YarnClient yarnClient = YarnClient.createYarnClient();
yarnClient.init(new Configuration(conf));
yarnClient.start();
String hostName = NetUtils.getHostname();
boolean verified = false;
String errorMessage = "";
while (!verified) {
List<ApplicationReport> apps = yarnClient.getApplications();
if (apps.size() == 0) {
Thread.sleep(10);
continue;
}
ApplicationReport appReport = apps.get(0);
if (appReport.getHost().equals("N/A")) {
Thread.sleep(10);
continue;
}
errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport.getHost() + "'. Expected rpc port to be '-1', was '" + appReport.getRpcPort() + "'.";
if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) {
verified = true;
}
if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED) {
break;
}
}
Assert.assertTrue(errorMessage, verified);
FileSystem fs = FileSystem.get(conf);
try {
new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--local_model_path", "src/test/resources/maas", "--hdfs_model_path", new Path(fs.getHomeDirectory(), "maas/dummy").toString(), "--num_instances", "1", "--memory", "100", "--mode", "ADD", "--log4j", "src/test/resources/log4j.properties" });
ServiceDiscoverer discoverer = new ServiceDiscoverer(this.client, config.getServiceRoot());
discoverer.start();
{
boolean passed = false;
for (int i = 0; i < 100; ++i) {
try {
List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0"));
if (endpoints != null && endpoints.size() == 1) {
LOG.trace("Found endpoints: " + endpoints.get(0));
String output = makeRESTcall(new URL(endpoints.get(0).getEndpoint().getUrl() + "/echo/casey"));
if (output.contains("casey")) {
passed = true;
break;
}
}
} catch (Exception e) {
}
Thread.sleep(2000);
}
Assert.assertTrue(passed);
}
{
List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0"));
Assert.assertNotNull(endpoints);
Assert.assertEquals(1, endpoints.size());
}
new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--num_instances", "1", "--mode", "REMOVE" });
{
boolean passed = false;
for (int i = 0; i < 100; ++i) {
try {
List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0"));
// ensure that the endpoint is dead.
if (endpoints == null || endpoints.size() == 0) {
passed = true;
break;
}
} catch (Exception e) {
}
Thread.sleep(2000);
}
Assert.assertTrue(passed);
}
} finally {
cleanup();
}
}
use of org.apache.metron.maas.discovery.ServiceDiscoverer in project metron by apache.
the class MaaSFunctions method createDiscoverer.
private static ServiceDiscoverer createDiscoverer(CuratorFramework client) throws Exception {
MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class);
ServiceDiscoverer discoverer = new ServiceDiscoverer(client, config.getServiceRoot());
discoverer.start();
return discoverer;
}
use of org.apache.metron.maas.discovery.ServiceDiscoverer in project metron by apache.
the class StellarMaaSIntegrationTest method setup.
@BeforeClass
public static void setup() throws Exception {
UnitTestHelper.setJavaLoggingLevel(WebApplicationImpl.class, Level.WARNING);
MockDGAModel.start(8282);
testZkServer = new TestingServer(true);
zookeeperUrl = testZkServer.getConnectString();
RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
client = CuratorFrameworkFactory.newClient(zookeeperUrl, retryPolicy);
client.start();
context = new Context.Builder().with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client).build();
MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class);
discoverer = new ServiceDiscoverer(client, config.getServiceRoot());
discoverer.start();
endpointUrl = new URL("http://localhost:8282");
ModelEndpoint endpoint = new ModelEndpoint();
{
endpoint.setName("dga");
endpoint.setContainerId("0");
Endpoint ep = new Endpoint();
ep.setUrl(endpointUrl.toString());
endpoint.setEndpoint(ep);
endpoint.setVersion("1.0");
}
;
ServiceInstanceBuilder<ModelEndpoint> builder = ServiceInstance.<ModelEndpoint>builder().address(endpointUrl.getHost()).id("0").name("dga").port(endpointUrl.getPort()).registrationTimeUTC(System.currentTimeMillis()).serviceType(ServiceType.STATIC).payload(endpoint);
final ServiceInstance<ModelEndpoint> instance = builder.build();
discoverer.getServiceDiscovery().registerService(instance);
// wait til the endpoint is installed...
for (int i = 0; i < 10; ++i) {
try {
Object o = discoverer.getEndpoint("dga");
if (o != null) {
break;
}
} catch (Exception e) {
}
Thread.sleep(1000);
}
}
Aggregations