Examples with ServiceDiscoverer org.apache.metron.maas.discovery.ServiceDiscoverer used on opensource projects

Example 1 with ServiceDiscoverer

use of org.apache.metron.maas.discovery.ServiceDiscoverer in project metron by apache.

the class MaaSHandler method start.

public void start() throws Exception {
    if (client == null) {
        RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
        client = CuratorFrameworkFactory.newClient(zkQuorum, retryPolicy);
        client.start();
    }
    config = ConfigUtil.INSTANCE.read(client, root, new MaaSConfig(), MaaSConfig.class);
    cache = new NodeCache(client, root);
    cache.getListenable().addListener(() -> {
        byte[] data = cache.getCurrentData().getData();
        Lock wLock = lock.writeLock();
        wLock.lock();
        try {
            config = _mapper.readValue(data, MaaSConfig.class);
        } finally {
            wLock.unlock();
        }
    });
    discoverer = new ServiceDiscoverer(client, config.getServiceRoot());
    discoverer.start();
}

Example 2 with ServiceDiscoverer

use of org.apache.metron.maas.discovery.ServiceDiscoverer in project metron by apache.

the class ModelSubmission method execute.

public void execute(FileSystem fs, String... argv) throws Exception {
    CommandLine cli = ModelSubmissionOptions.parse(new PosixParser(), argv);
    if (ModelSubmissionOptions.LOG4J_PROPERTIES.has(cli)) {
        Log4jPropertyHelper.updateLog4jConfiguration(ModelSubmission.class, ModelSubmissionOptions.LOG4J_PROPERTIES.get(cli));
    }
    ModelRequest request = null;
    CuratorFramework client = null;
    try {
        RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
        client = CuratorFrameworkFactory.newClient(ModelSubmissionOptions.ZK_QUORUM.get(cli), retryPolicy);
        client.start();
        MaaSConfig config = ConfigUtil.INSTANCE.read(client, ModelSubmissionOptions.ZK_ROOT.get(cli, "/metron/maas/config"), new MaaSConfig(), MaaSConfig.class);
        String mode = ModelSubmissionOptions.MODE.get(cli);
        if (mode.equalsIgnoreCase("ADD")) {
            request = new ModelRequest() {

                {
                    setName(ModelSubmissionOptions.NAME.get(cli));
                    setAction(Action.ADD);
                    setVersion(ModelSubmissionOptions.VERSION.get(cli));
                    setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli)));
                    setMemory(Integer.parseInt(ModelSubmissionOptions.MEMORY.get(cli)));
                    setPath(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli));
                }
            };
        } else if (mode.equalsIgnoreCase("REMOVE")) {
            request = new ModelRequest() {

                {
                    setName(ModelSubmissionOptions.NAME.get(cli));
                    setAction(Action.REMOVE);
                    setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli)));
                    setVersion(ModelSubmissionOptions.VERSION.get(cli));
                }
            };
        } else if (mode.equalsIgnoreCase("LIST")) {
            String name = ModelSubmissionOptions.NAME.get(cli, null);
            String version = ModelSubmissionOptions.VERSION.get(cli, null);
            ServiceDiscoverer serviceDiscoverer = new ServiceDiscoverer(client, config.getServiceRoot());
            Model model = new Model(name, version);
            Map<Model, List<ModelEndpoint>> endpoints = serviceDiscoverer.listEndpoints(model);
            for (Map.Entry<Model, List<ModelEndpoint>> kv : endpoints.entrySet()) {
                String modelTitle = "Model " + kv.getKey().getName() + " @ " + kv.getKey().getVersion();
                System.out.println(modelTitle);
                for (ModelEndpoint endpoint : kv.getValue()) {
                    System.out.println(endpoint);
                }
            }
        }
        if (ModelSubmissionOptions.LOCAL_MODEL_PATH.has(cli)) {
            File localDir = new File(ModelSubmissionOptions.LOCAL_MODEL_PATH.get(cli));
            Path hdfsPath = new Path(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli));
            updateHDFS(fs, localDir, hdfsPath);
        }
        Queue<ModelRequest> queue = config.createQueue(ImmutableMap.of(ZKQueue.ZK_CLIENT, client));
        queue.enqueue(request);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}

Example 3 with ServiceDiscoverer

use of org.apache.metron.maas.discovery.ServiceDiscoverer in project metron by apache.

the class MaasIntegrationTest method testDSShell.

public void testDSShell(boolean haveDomain) throws Exception {
    MaaSConfig config = new MaaSConfig() {

        {
            setServiceRoot("/maas/service");
            setQueueConfig(new HashMap<String, Object>() {

                {
                    put(ZKQueue.ZK_PATH, "/maas/queue");
                }
            });
        }
    };
    String configRoot = "/maas/config";
    byte[] configData = ConfigUtil.INSTANCE.toBytes(config);
    try {
        client.setData().forPath(configRoot, configData);
    } catch (KeeperException.NoNodeException e) {
        client.create().creatingParentsIfNeeded().forPath(configRoot, configData);
    }
    String[] args = { "--jar", yarnComponent.getAppMasterJar(), "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--master_memory", "512", "--master_vcores", "2" };
    if (haveDomain) {
        String[] domainArgs = { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group", "--modify_acls", "writer_user writer_group", "--create" };
        List<String> argsList = new ArrayList<String>(Arrays.asList(args));
        argsList.addAll(Arrays.asList(domainArgs));
        args = argsList.toArray(new String[argsList.size()]);
    }
    YarnConfiguration conf = yarnComponent.getConfig();
    LOG.info("Initializing DS Client");
    final Client client = new Client(new Configuration(conf));
    boolean initSuccess = client.init(args);
    Assert.assertTrue(initSuccess);
    LOG.info("Running DS Client");
    final AtomicBoolean result = new AtomicBoolean(false);
    Thread t = new Thread() {

        @Override
        public void run() {
            try {
                result.set(client.run());
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    };
    t.start();
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(new Configuration(conf));
    yarnClient.start();
    String hostName = NetUtils.getHostname();
    boolean verified = false;
    String errorMessage = "";
    while (!verified) {
        List<ApplicationReport> apps = yarnClient.getApplications();
        if (apps.size() == 0) {
            Thread.sleep(10);
            continue;
        }
        ApplicationReport appReport = apps.get(0);
        if (appReport.getHost().equals("N/A")) {
            Thread.sleep(10);
            continue;
        }
        errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport.getHost() + "'. Expected rpc port to be '-1', was '" + appReport.getRpcPort() + "'.";
        if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) {
            verified = true;
        }
        if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED) {
            break;
        }
    }
    Assert.assertTrue(errorMessage, verified);
    FileSystem fs = FileSystem.get(conf);
    try {
        new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--local_model_path", "src/test/resources/maas", "--hdfs_model_path", new Path(fs.getHomeDirectory(), "maas/dummy").toString(), "--num_instances", "1", "--memory", "100", "--mode", "ADD", "--log4j", "src/test/resources/log4j.properties" });
        ServiceDiscoverer discoverer = new ServiceDiscoverer(this.client, config.getServiceRoot());
        discoverer.start();
        {
            boolean passed = false;
            for (int i = 0; i < 100; ++i) {
                try {
                    List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0"));
                    if (endpoints != null && endpoints.size() == 1) {
                        LOG.trace("Found endpoints: " + endpoints.get(0));
                        String output = makeRESTcall(new URL(endpoints.get(0).getEndpoint().getUrl() + "/echo/casey"));
                        if (output.contains("casey")) {
                            passed = true;
                            break;
                        }
                    }
                } catch (Exception e) {
                }
                Thread.sleep(2000);
            }
            Assert.assertTrue(passed);
        }
        {
            List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0"));
            Assert.assertNotNull(endpoints);
            Assert.assertEquals(1, endpoints.size());
        }
        new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--num_instances", "1", "--mode", "REMOVE" });
        {
            boolean passed = false;
            for (int i = 0; i < 100; ++i) {
                try {
                    List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0"));
                    // ensure that the endpoint is dead.
                    if (endpoints == null || endpoints.size() == 0) {
                        passed = true;
                        break;
                    }
                } catch (Exception e) {
                }
                Thread.sleep(2000);
            }
            Assert.assertTrue(passed);
        }
    } finally {
        cleanup();
    }
}

Example 4 with ServiceDiscoverer

use of org.apache.metron.maas.discovery.ServiceDiscoverer in project metron by apache.

the class MaaSFunctions method createDiscoverer.

private static ServiceDiscoverer createDiscoverer(CuratorFramework client) throws Exception {
    MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class);
    ServiceDiscoverer discoverer = new ServiceDiscoverer(client, config.getServiceRoot());
    discoverer.start();
    return discoverer;
}

Example 5 with ServiceDiscoverer

use of org.apache.metron.maas.discovery.ServiceDiscoverer in project metron by apache.

the class StellarMaaSIntegrationTest method setup.

@BeforeClass
public static void setup() throws Exception {
    UnitTestHelper.setJavaLoggingLevel(WebApplicationImpl.class, Level.WARNING);
    MockDGAModel.start(8282);
    testZkServer = new TestingServer(true);
    zookeeperUrl = testZkServer.getConnectString();
    RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
    client = CuratorFrameworkFactory.newClient(zookeeperUrl, retryPolicy);
    client.start();
    context = new Context.Builder().with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client).build();
    MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class);
    discoverer = new ServiceDiscoverer(client, config.getServiceRoot());
    discoverer.start();
    endpointUrl = new URL("http://localhost:8282");
    ModelEndpoint endpoint = new ModelEndpoint();
    {
        endpoint.setName("dga");
        endpoint.setContainerId("0");
        Endpoint ep = new Endpoint();
        ep.setUrl(endpointUrl.toString());
        endpoint.setEndpoint(ep);
        endpoint.setVersion("1.0");
    }
    ;
    ServiceInstanceBuilder<ModelEndpoint> builder = ServiceInstance.<ModelEndpoint>builder().address(endpointUrl.getHost()).id("0").name("dga").port(endpointUrl.getPort()).registrationTimeUTC(System.currentTimeMillis()).serviceType(ServiceType.STATIC).payload(endpoint);
    final ServiceInstance<ModelEndpoint> instance = builder.build();
    discoverer.getServiceDiscovery().registerService(instance);
    // wait til the endpoint is installed...
    for (int i = 0; i < 10; ++i) {
        try {
            Object o = discoverer.getEndpoint("dga");
            if (o != null) {
                break;
            }
        } catch (Exception e) {
        }
        Thread.sleep(1000);
    }
}