Examples with Model org.apache.metron.maas.config.Model used on opensource projects

Example 1 with Model

use of org.apache.metron.maas.config.Model in project metron by apache.

the class MaasIntegrationTest method testDSShell.

public void testDSShell(boolean haveDomain) throws Exception {
    MaaSConfig config = new MaaSConfig() {

        {
            setServiceRoot("/maas/service");
            setQueueConfig(new HashMap<String, Object>() {

                {
                    put(ZKQueue.ZK_PATH, "/maas/queue");
                }
            });
        }
    };
    String configRoot = "/maas/config";
    byte[] configData = ConfigUtil.INSTANCE.toBytes(config);
    try {
        client.setData().forPath(configRoot, configData);
    } catch (KeeperException.NoNodeException e) {
        client.create().creatingParentsIfNeeded().forPath(configRoot, configData);
    }
    String[] args = { "--jar", yarnComponent.getAppMasterJar(), "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--master_memory", "512", "--master_vcores", "2" };
    if (haveDomain) {
        String[] domainArgs = { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group", "--modify_acls", "writer_user writer_group", "--create" };
        List<String> argsList = new ArrayList<String>(Arrays.asList(args));
        argsList.addAll(Arrays.asList(domainArgs));
        args = argsList.toArray(new String[argsList.size()]);
    }
    YarnConfiguration conf = yarnComponent.getConfig();
    LOG.info("Initializing DS Client");
    final Client client = new Client(new Configuration(conf));
    boolean initSuccess = client.init(args);
    assertTrue(initSuccess);
    LOG.info("Running DS Client");
    final AtomicBoolean result = new AtomicBoolean(false);
    Thread t = new Thread() {

        @Override
        public void run() {
            try {
                result.set(client.run());
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    };
    t.start();
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(new Configuration(conf));
    yarnClient.start();
    String hostName = NetUtils.getHostname();
    boolean verified = false;
    String errorMessage = "";
    while (!verified) {
        List<ApplicationReport> apps = yarnClient.getApplications();
        if (apps.size() == 0) {
            Thread.sleep(10);
            continue;
        }
        ApplicationReport appReport = apps.get(0);
        if (appReport.getHost().equals("N/A")) {
            Thread.sleep(10);
            continue;
        }
        errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport.getHost() + "'. Expected rpc port to be '-1', was '" + appReport.getRpcPort() + "'.";
        if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) {
            verified = true;
        }
        if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED) {
            break;
        }
    }
    assertTrue(verified, errorMessage);
    FileSystem fs = FileSystem.get(conf);
    try {
        new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--local_model_path", "src/test/resources/maas", "--hdfs_model_path", new Path(fs.getHomeDirectory(), "maas/dummy").toString(), "--num_instances", "1", "--memory", "100", "--mode", "ADD", "--log4j", "src/test/resources/log4j.properties" });
        ServiceDiscoverer discoverer = new ServiceDiscoverer(this.client, config.getServiceRoot());
        discoverer.start();
        {
            boolean passed = false;
            for (int i = 0; i < 100; ++i) {
                try {
                    List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0"));
                    if (endpoints != null && endpoints.size() == 1) {
                        LOG.trace("Found endpoints: " + endpoints.get(0));
                        String output = makeRESTcall(new URL(endpoints.get(0).getEndpoint().getUrl() + "/echo/casey"));
                        if (output.contains("casey")) {
                            passed = true;
                            break;
                        }
                    }
                } catch (Exception e) {
                }
                Thread.sleep(2000);
            }
            assertTrue(passed);
        }
        {
            List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0"));
            assertNotNull(endpoints);
            assertEquals(1, endpoints.size());
        }
        new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--num_instances", "1", "--mode", "REMOVE" });
        {
            boolean passed = false;
            for (int i = 0; i < 100; ++i) {
                try {
                    List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0"));
                    // ensure that the endpoint is dead.
                    if (endpoints == null || endpoints.size() == 0) {
                        passed = true;
                        break;
                    }
                } catch (Exception e) {
                }
                Thread.sleep(2000);
            }
            assertTrue(passed);
        }
    } finally {
        cleanup();
    }
}

Example 2 with Model

use of org.apache.metron.maas.config.Model in project metron by apache.

the class ContainerTracker method getList.

public List<Container> getList(ModelRequest request) {
    synchronized (acceptedContainersByResource) {
        List<Container> containers = launchedContainers.get(new Model(request.getName(), request.getVersion()));
        if (containers == null) {
            containers = new ArrayList<>();
            launchedContainers.put(new Model(request.getName(), request.getVersion()), containers);
        }
        return containers;
    }
}

Example 3 with Model

use of org.apache.metron.maas.config.Model in project metron by apache.

the class ServiceDiscoveryIntegrationTest method testDiscovery.

@Test
public void testDiscovery() throws Exception {
    // register some models
    AtomicInteger containerId = new AtomicInteger(0);
    registerService("casey", "3.14159", containerId);
    registerService("casey", "3.14159", containerId);
    registerService("casey", "3.14159", containerId);
    registerService("casey", "3.1416", containerId);
    // wait for zk to percolate the changes.
    Thread.sleep(2000);
    assertEquals(3, discoverer.getEndpoints(new Model("casey", "3.14159")).size());
    assertEquals(1, discoverer.getEndpoints(new Model("casey", "3.1416")).size());
    assertEquals(0, discoverer.getEndpoints(new Model("casey", "3.17")).size());
    discoverer.unregisterByContainer("1");
    Thread.sleep(2000);
    assertEquals(2, discoverer.getEndpoints(new Model("casey", "3.14159")).size());
    assertEquals(1, discoverer.getEndpoints(new Model("casey", "3.1416")).size());
    assertEquals(0, discoverer.getEndpoints(new Model("casey", "3.17")).size());
    assertEquals(2, discoverer.listEndpoints(new Model("casey", null)).keySet().size());
    assertEquals(1, discoverer.listEndpoints(new Model("casey", "3.1416")).keySet().size());
    assertEquals(1, discoverer.listEndpoints(new Model("casey", "3.1416")).get(new Model("casey", "3.1416")).size());
    assertEquals("4", discoverer.listEndpoints(new Model("casey", "3.1416")).get(new Model("casey", "3.1416")).get(0).getContainerId());
    assertEquals(0, discoverer.listEndpoints(new Model("casey", "3.17")).keySet().size());
    assertEquals(0, discoverer.listEndpoints(new Model("dummy", null)).keySet().size());
}

Example 4 with Model

use of org.apache.metron.maas.config.Model in project metron by apache.

the class ServiceDiscoverer method updateState.

private void updateState() {
    Map<Model, List<ModelEndpoint>> state = new HashMap<>();
    Map<String, String> modelToVersion = new HashMap<>();
    Map<String, ServiceInstance<ModelEndpoint>> containerToEndpoint = new HashMap<>();
    try {
        for (String name : serviceDiscovery.queryForNames()) {
            for (ServiceInstance<ModelEndpoint> endpoint : serviceDiscovery.queryForInstances(name)) {
                ModelEndpoint ep = endpoint.getPayload();
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Found model endpoint " + ep);
                }
                // initialize to the existing current version, defaulting to this version
                String currentVersion = modelToVersion.getOrDefault(ep.getName(), ep.getVersion());
                // if the version for this endpont is greater than the current version, then use this one
                // otherwise use the version we know about.
                // essentially it's equivalent to currentVersion = max(ep.getVersion(), currentVersion)
                currentVersion = currentVersion.compareTo(ep.getVersion()) < 0 ? ep.getVersion() : currentVersion;
                modelToVersion.put(ep.getName(), currentVersion);
                containerToEndpoint.put(ep.getContainerId(), endpoint);
                Model model = new Model(ep.getName(), ep.getVersion());
                List<ModelEndpoint> endpoints = state.get(model);
                if (endpoints == null) {
                    endpoints = new ArrayList<>();
                    state.put(model, endpoints);
                }
                endpoints.add(ep);
            }
        }
        rwLock.writeLock().lock();
        try {
            this.modelToCurrentVersion = modelToVersion;
            this.state = state;
            this.containerToEndpoint = containerToEndpoint;
            if (LOG.isDebugEnabled()) {
                LOG.debug("Containers found: " + containerToEndpoint);
            }
        } finally {
            rwLock.writeLock().unlock();
        }
    } catch (Exception e) {
        LOG.error(e.getMessage(), e);
    } finally {
    }
}