Examples with MaaSConfig org.apache.metron.maas.config.MaaSConfig used on opensource projects

Example 1 with MaaSConfig

use of org.apache.metron.maas.config.MaaSConfig in project metron by apache.

the class MaaSHandler method start.

public void start() throws Exception {
    if (client == null) {
        RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
        client = CuratorFrameworkFactory.newClient(zkQuorum, retryPolicy);
        client.start();
    }
    config = ConfigUtil.INSTANCE.read(client, root, new MaaSConfig(), MaaSConfig.class);
    cache = new NodeCache(client, root);
    cache.getListenable().addListener(() -> {
        byte[] data = cache.getCurrentData().getData();
        Lock wLock = lock.writeLock();
        wLock.lock();
        try {
            config = _mapper.readValue(data, MaaSConfig.class);
        } finally {
            wLock.unlock();
        }
    });
    discoverer = new ServiceDiscoverer(client, config.getServiceRoot());
    discoverer.start();
}

Example 2 with MaaSConfig

use of org.apache.metron.maas.config.MaaSConfig in project metron by apache.

the class MaasIntegrationTest method testDSShell.

public void testDSShell(boolean haveDomain) throws Exception {
    MaaSConfig config = new MaaSConfig() {

        {
            setServiceRoot("/maas/service");
            setQueueConfig(new HashMap<String, Object>() {

                {
                    put(ZKQueue.ZK_PATH, "/maas/queue");
                }
            });
        }
    };
    String configRoot = "/maas/config";
    byte[] configData = ConfigUtil.INSTANCE.toBytes(config);
    try {
        client.setData().forPath(configRoot, configData);
    } catch (KeeperException.NoNodeException e) {
        client.create().creatingParentsIfNeeded().forPath(configRoot, configData);
    }
    String[] args = { "--jar", yarnComponent.getAppMasterJar(), "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--master_memory", "512", "--master_vcores", "2" };
    if (haveDomain) {
        String[] domainArgs = { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group", "--modify_acls", "writer_user writer_group", "--create" };
        List<String> argsList = new ArrayList<String>(Arrays.asList(args));
        argsList.addAll(Arrays.asList(domainArgs));
        args = argsList.toArray(new String[argsList.size()]);
    }
    YarnConfiguration conf = yarnComponent.getConfig();
    LOG.info("Initializing DS Client");
    final Client client = new Client(new Configuration(conf));
    boolean initSuccess = client.init(args);
    Assert.assertTrue(initSuccess);
    LOG.info("Running DS Client");
    final AtomicBoolean result = new AtomicBoolean(false);
    Thread t = new Thread() {

        @Override
        public void run() {
            try {
                result.set(client.run());
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    };
    t.start();
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(new Configuration(conf));
    yarnClient.start();
    String hostName = NetUtils.getHostname();
    boolean verified = false;
    String errorMessage = "";
    while (!verified) {
        List<ApplicationReport> apps = yarnClient.getApplications();
        if (apps.size() == 0) {
            Thread.sleep(10);
            continue;
        }
        ApplicationReport appReport = apps.get(0);
        if (appReport.getHost().equals("N/A")) {
            Thread.sleep(10);
            continue;
        }
        errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport.getHost() + "'. Expected rpc port to be '-1', was '" + appReport.getRpcPort() + "'.";
        if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) {
            verified = true;
        }
        if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED) {
            break;
        }
    }
    Assert.assertTrue(errorMessage, verified);
    FileSystem fs = FileSystem.get(conf);
    try {
        new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--local_model_path", "src/test/resources/maas", "--hdfs_model_path", new Path(fs.getHomeDirectory(), "maas/dummy").toString(), "--num_instances", "1", "--memory", "100", "--mode", "ADD", "--log4j", "src/test/resources/log4j.properties" });
        ServiceDiscoverer discoverer = new ServiceDiscoverer(this.client, config.getServiceRoot());
        discoverer.start();
        {
            boolean passed = false;
            for (int i = 0; i < 100; ++i) {
                try {
                    List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0"));
                    if (endpoints != null && endpoints.size() == 1) {
                        LOG.trace("Found endpoints: " + endpoints.get(0));
                        String output = makeRESTcall(new URL(endpoints.get(0).getEndpoint().getUrl() + "/echo/casey"));
                        if (output.contains("casey")) {
                            passed = true;
                            break;
                        }
                    }
                } catch (Exception e) {
                }
                Thread.sleep(2000);
            }
            Assert.assertTrue(passed);
        }
        {
            List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0"));
            Assert.assertNotNull(endpoints);
            Assert.assertEquals(1, endpoints.size());
        }
        new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--num_instances", "1", "--mode", "REMOVE" });
        {
            boolean passed = false;
            for (int i = 0; i < 100; ++i) {
                try {
                    List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0"));
                    // ensure that the endpoint is dead.
                    if (endpoints == null || endpoints.size() == 0) {
                        passed = true;
                        break;
                    }
                } catch (Exception e) {
                }
                Thread.sleep(2000);
            }
            Assert.assertTrue(passed);
        }
    } finally {
        cleanup();
    }
}

Example 3 with MaaSConfig

use of org.apache.metron.maas.config.MaaSConfig in project metron by apache.

the class MaaSFunctions method createDiscoverer.

private static ServiceDiscoverer createDiscoverer(CuratorFramework client) throws Exception {
    MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class);
    ServiceDiscoverer discoverer = new ServiceDiscoverer(client, config.getServiceRoot());
    discoverer.start();
    return discoverer;
}

Example 4 with MaaSConfig

use of org.apache.metron.maas.config.MaaSConfig in project metron by apache.

the class Runner method main.

public static void main(String... argv) throws Exception {
    CommandLine cli = RunnerOptions.parse(new PosixParser(), argv);
    String zkQuorum = RunnerOptions.ZK_QUORUM.get(cli);
    String zkRoot = RunnerOptions.ZK_ROOT.get(cli);
    String script = RunnerOptions.SCRIPT.get(cli);
    String name = RunnerOptions.NAME.get(cli);
    String version = RunnerOptions.VERSION.get(cli);
    String containerId = RunnerOptions.CONTAINER_ID.get(cli);
    String hostname = RunnerOptions.HOSTNAME.get(cli);
    CuratorFramework client = null;
    LOG.error("Running script " + script);
    LOG.info("Local Directory Contents");
    for (File f : new File(".").listFiles()) {
        LOG.info("  " + f.getName());
    }
    try {
        RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
        client = CuratorFrameworkFactory.newClient(zkQuorum, retryPolicy);
        client.start();
        MaaSConfig config = ConfigUtil.INSTANCE.read(client, zkRoot, new MaaSConfig(), MaaSConfig.class);
        JsonInstanceSerializer<ModelEndpoint> serializer = new JsonInstanceSerializer<>(ModelEndpoint.class);
        try {
            serviceDiscovery = ServiceDiscoveryBuilder.builder(ModelEndpoint.class).client(client).basePath(config.getServiceRoot()).serializer(serializer).build();
        } finally {
        }
        LOG.info("Created service @ " + config.getServiceRoot());
        serviceDiscovery.start();
        File cwd = new File(script).getParentFile();
        File scriptFile = new File(cwd, script);
        if (scriptFile.exists() && !scriptFile.canExecute()) {
            scriptFile.setExecutable(true);
        }
        final String cmd = scriptFile.getAbsolutePath();
        try {
            p = new ProcessBuilder(cmd).directory(cwd).start();
        } catch (Exception e) {
            LOG.info("Unable to execute " + cmd + " from " + new File(".").getAbsolutePath());
            LOG.error(e.getMessage(), e);
            throw new IllegalStateException(e.getMessage(), e);
        }
        try {
            LOG.info("Started " + cmd);
            Endpoint ep = readEndpoint(cwd);
            URL endpointUrl = correctLocalUrl(hostname, ep.getUrl());
            ep.setUrl(endpointUrl.toString());
            LOG.info("Read endpoint " + ep);
            ModelEndpoint endpoint = new ModelEndpoint();
            {
                endpoint.setName(name);
                endpoint.setContainerId(containerId);
                endpoint.setEndpoint(ep);
                endpoint.setVersion(version);
            }
            ;
            ServiceInstanceBuilder<ModelEndpoint> builder = ServiceInstance.<ModelEndpoint>builder().address(endpointUrl.getHost()).id(containerId).name(name).port(endpointUrl.getPort()).registrationTimeUTC(System.currentTimeMillis()).serviceType(ServiceType.STATIC).payload(endpoint);
            final ServiceInstance<ModelEndpoint> instance = builder.build();
            try {
                LOG.info("Installing service instance: " + instance + " at " + serviceDiscovery);
                serviceDiscovery.registerService(instance);
                LOG.info("Installed instance " + name + ":" + version + "@" + endpointUrl);
            } catch (Throwable t) {
                LOG.error("Unable to install instance " + name + ":" + version + "@" + endpointUrl, t);
            }
            Runtime.getRuntime().addShutdownHook(new Thread() {

                @Override
                public void run() {
                    LOG.info("KILLING CONTAINER PROCESS...");
                    if (p != null) {
                        LOG.info("Process destroyed forcibly");
                        p.destroyForcibly();
                    }
                }
            });
        } finally {
            if (p.waitFor() != 0) {
                String stderr = Joiner.on("\n").join(IOUtils.readLines(p.getErrorStream()));
                String stdout = Joiner.on("\n").join(IOUtils.readLines(p.getInputStream()));
                throw new IllegalStateException("Unable to execute " + script + ".  Stderr is: " + stderr + "\nStdout is: " + stdout);
            }
        }
    } finally {
        if (serviceDiscovery != null) {
            CloseableUtils.closeQuietly(serviceDiscovery);
        }
        if (client != null) {
            CloseableUtils.closeQuietly(client);
        }
    }
}

Example 5 with MaaSConfig

use of org.apache.metron.maas.config.MaaSConfig in project metron by apache.

the class StellarMaaSIntegrationTest method setup.

@BeforeClass
public static void setup() throws Exception {
    UnitTestHelper.setJavaLoggingLevel(WebApplicationImpl.class, Level.WARNING);
    MockDGAModel.start(8282);
    testZkServer = new TestingServer(true);
    zookeeperUrl = testZkServer.getConnectString();
    RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
    client = CuratorFrameworkFactory.newClient(zookeeperUrl, retryPolicy);
    client.start();
    context = new Context.Builder().with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client).build();
    MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class);
    discoverer = new ServiceDiscoverer(client, config.getServiceRoot());
    discoverer.start();
    endpointUrl = new URL("http://localhost:8282");
    ModelEndpoint endpoint = new ModelEndpoint();
    {
        endpoint.setName("dga");
        endpoint.setContainerId("0");
        Endpoint ep = new Endpoint();
        ep.setUrl(endpointUrl.toString());
        endpoint.setEndpoint(ep);
        endpoint.setVersion("1.0");
    }
    ;
    ServiceInstanceBuilder<ModelEndpoint> builder = ServiceInstance.<ModelEndpoint>builder().address(endpointUrl.getHost()).id("0").name("dga").port(endpointUrl.getPort()).registrationTimeUTC(System.currentTimeMillis()).serviceType(ServiceType.STATIC).payload(endpoint);
    final ServiceInstance<ModelEndpoint> instance = builder.build();
    discoverer.getServiceDiscovery().registerService(instance);
    // wait til the endpoint is installed...
    for (int i = 0; i < 10; ++i) {
        try {
            Object o = discoverer.getEndpoint("dga");
            if (o != null) {
                break;
            }
        } catch (Exception e) {
        }
        Thread.sleep(1000);
    }
}