Examples with TopologyValidator org.apache.knox.gateway.topology.validation.TopologyValidator used on opensource projects

Example 1 with TopologyValidator

use of org.apache.knox.gateway.topology.validation.TopologyValidator in project knox by apache.

the class SimpleDescriptorHandlerTest method testSimpleDescriptorHandler.

/**
 * KNOX-1006
 *
 * N.B. This test depends on the PropertiesFileServiceDiscovery extension being configured:
 *             org.apache.knox.gateway.topology.discovery.test.extension.PropertiesFileServiceDiscovery
 */
@Test
public void testSimpleDescriptorHandler() throws Exception {
    final String type = "PROPERTIES_FILE";
    final String clusterName = "dummy";
    // Create a properties file to be the source of service discovery details for this test
    final File discoveryConfig = File.createTempFile(getClass().getName() + "_discovery-config", ".properties");
    final String address = discoveryConfig.getAbsolutePath();
    final Properties DISCOVERY_PROPERTIES = new Properties();
    DISCOVERY_PROPERTIES.setProperty(clusterName + ".name", clusterName);
    DISCOVERY_PROPERTIES.setProperty(clusterName + ".NAMENODE", "hdfs://namenodehost:8020");
    DISCOVERY_PROPERTIES.setProperty(clusterName + ".JOBTRACKER", "rpc://jobtrackerhostname:8050");
    DISCOVERY_PROPERTIES.setProperty(clusterName + ".WEBHDFS", "http://webhdfshost:1234");
    DISCOVERY_PROPERTIES.setProperty(clusterName + ".WEBHCAT", "http://webhcathost:50111/templeton");
    DISCOVERY_PROPERTIES.setProperty(clusterName + ".OOZIE", "http://ooziehost:11000/oozie");
    DISCOVERY_PROPERTIES.setProperty(clusterName + ".WEBHBASE", "http://webhbasehost:1234");
    DISCOVERY_PROPERTIES.setProperty(clusterName + ".HIVE", "http://hivehostname:10001/clipath");
    DISCOVERY_PROPERTIES.setProperty(clusterName + ".RESOURCEMANAGER", "http://remanhost:8088/ws");
    try {
        DISCOVERY_PROPERTIES.store(new FileOutputStream(discoveryConfig), null);
    } catch (FileNotFoundException e) {
        fail(e.getMessage());
    }
    final Map<String, List<String>> serviceURLs = new HashMap<>();
    serviceURLs.put("NAMENODE", null);
    serviceURLs.put("JOBTRACKER", null);
    serviceURLs.put("WEBHDFS", null);
    serviceURLs.put("WEBHCAT", null);
    serviceURLs.put("OOZIE", null);
    serviceURLs.put("WEBHBASE", null);
    serviceURLs.put("HIVE", null);
    serviceURLs.put("RESOURCEMANAGER", null);
    serviceURLs.put("AMBARIUI", Collections.singletonList("http://c6401.ambari.apache.org:8080"));
    serviceURLs.put("KNOXSSO", null);
    // Write the externalized provider config to a temp file
    File providerConfig = new File(System.getProperty("java.io.tmpdir"), "ambari-cluster-policy.xml");
    FileUtils.write(providerConfig, TEST_PROVIDER_CONFIG);
    File topologyFile = null;
    try {
        File destDir = new File(System.getProperty("java.io.tmpdir")).getCanonicalFile();
        Map<String, Map<String, String>> serviceParameters = new HashMap<>();
        Map<String, String> knoxssoParams = new HashMap<>();
        knoxssoParams.put("knoxsso.cookie.secure.only", "true");
        knoxssoParams.put("knoxsso.token.ttl", "100000");
        serviceParameters.put("KNOXSSO", knoxssoParams);
        GatewayConfig gc = EasyMock.createNiceMock(GatewayConfig.class);
        EasyMock.replay(gc);
        // Mock out the simple descriptor
        SimpleDescriptor testDescriptor = EasyMock.createNiceMock(SimpleDescriptor.class);
        EasyMock.expect(testDescriptor.getName()).andReturn("mysimpledescriptor").anyTimes();
        EasyMock.expect(testDescriptor.getDiscoveryAddress()).andReturn(address).anyTimes();
        EasyMock.expect(testDescriptor.getDiscoveryType()).andReturn(type).anyTimes();
        EasyMock.expect(testDescriptor.getDiscoveryUser()).andReturn(null).anyTimes();
        EasyMock.expect(testDescriptor.getProviderConfig()).andReturn(providerConfig.getAbsolutePath()).anyTimes();
        EasyMock.expect(testDescriptor.getClusterName()).andReturn(clusterName).anyTimes();
        List<SimpleDescriptor.Service> serviceMocks = new ArrayList<>();
        for (String serviceName : serviceURLs.keySet()) {
            SimpleDescriptor.Service svc = EasyMock.createNiceMock(SimpleDescriptor.Service.class);
            EasyMock.expect(svc.getName()).andReturn(serviceName).anyTimes();
            EasyMock.expect(svc.getVersion()).andReturn("WEBHDFS".equals(serviceName) ? "2.4.0" : null).anyTimes();
            EasyMock.expect(svc.getURLs()).andReturn(serviceURLs.get(serviceName)).anyTimes();
            EasyMock.expect(svc.getParams()).andReturn(serviceParameters.get(serviceName)).anyTimes();
            EasyMock.replay(svc);
            serviceMocks.add(svc);
        }
        EasyMock.expect(testDescriptor.getServices()).andReturn(serviceMocks).anyTimes();
        EasyMock.replay(testDescriptor);
        // Invoke the simple descriptor handler
        Map<String, File> files = SimpleDescriptorHandler.handle(gc, testDescriptor, // simple desc co-located with provider config
        providerConfig.getParentFile(), destDir);
        topologyFile = files.get("topology");
        // Validate the resulting topology descriptor
        assertTrue(topologyFile.exists());
        // Validate the topology descriptor's correctness
        TopologyValidator validator = new TopologyValidator(topologyFile.getAbsolutePath());
        if (!validator.validateTopology()) {
            throw new SAXException(validator.getErrorString());
        }
        XPathFactory xPathfactory = XPathFactory.newInstance();
        XPath xpath = xPathfactory.newXPath();
        // Parse the topology descriptor
        Document topologyXml = XmlUtils.readXml(topologyFile);
        // KNOX-1105 Mark generated topology files
        assertThat("Expected the \"generated\" marker element in the topology XML, with value of \"true\".", topologyXml, hasXPath("/topology/generated", is("true")));
        // Validate the provider configuration
        Node gatewayNode = (Node) xpath.compile("/topology/gateway").evaluate(topologyXml, XPathConstants.NODE);
        ProviderConfiguration testProviderConfiguration = ProviderConfigurationParser.parseXML(new ByteArrayInputStream(TEST_PROVIDER_CONFIG.getBytes()));
        validateGeneratedProviderConfiguration(testProviderConfiguration, gatewayNode);
        // Validate the service declarations
        Map<String, List<String>> topologyServiceURLs = new HashMap<>();
        NodeList serviceNodes = (NodeList) xpath.compile("/topology/service").evaluate(topologyXml, XPathConstants.NODESET);
        for (int serviceNodeIndex = 0; serviceNodeIndex < serviceNodes.getLength(); serviceNodeIndex++) {
            Node serviceNode = serviceNodes.item(serviceNodeIndex);
            // Validate the role
            Node roleNode = (Node) xpath.compile("role/text()").evaluate(serviceNode, XPathConstants.NODE);
            assertNotNull(roleNode);
            String role = roleNode.getNodeValue();
            // Validate the explicit version for the WEBHDFS service
            if ("WEBHDFS".equals(role)) {
                Node versionNode = (Node) xpath.compile("version/text()").evaluate(serviceNode, XPathConstants.NODE);
                assertNotNull(versionNode);
                String version = versionNode.getNodeValue();
                assertEquals("2.4.0", version);
            }
            // Validate the URLs
            NodeList urlNodes = (NodeList) xpath.compile("url/text()").evaluate(serviceNode, XPathConstants.NODESET);
            for (int urlNodeIndex = 0; urlNodeIndex < urlNodes.getLength(); urlNodeIndex++) {
                Node urlNode = urlNodes.item(urlNodeIndex);
                assertNotNull(urlNode);
                String url = urlNode.getNodeValue();
                // If the service should have a URL (some don't require it)
                if (serviceURLs.containsKey(role)) {
                    assertNotNull("Declared service should have a URL.", url);
                    if (!topologyServiceURLs.containsKey(role)) {
                        topologyServiceURLs.put(role, new ArrayList<>());
                    }
                    // Add it for validation later
                    topologyServiceURLs.get(role).add(url);
                }
            }
            // If params were declared in the descriptor, then validate them in the resulting topology file
            Map<String, String> params = serviceParameters.get(role);
            if (params != null) {
                NodeList paramNodes = (NodeList) xpath.compile("param").evaluate(serviceNode, XPathConstants.NODESET);
                for (int paramNodeIndex = 0; paramNodeIndex < paramNodes.getLength(); paramNodeIndex++) {
                    Node paramNode = paramNodes.item(paramNodeIndex);
                    String paramName = (String) xpath.compile("name/text()").evaluate(paramNode, XPathConstants.STRING);
                    String paramValue = (String) xpath.compile("value/text()").evaluate(paramNode, XPathConstants.STRING);
                    assertTrue(params.keySet().contains(paramName));
                    assertEquals(params.get(paramName), paramValue);
                }
            }
        }
        assertEquals("Unexpected number of service declarations.", (serviceURLs.size() - 1), topologyServiceURLs.size());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        providerConfig.delete();
        discoveryConfig.delete();
        if (topologyFile != null) {
            topologyFile.delete();
        }
    }
}

Example 2 with TopologyValidator

use of org.apache.knox.gateway.topology.validation.TopologyValidator in project knox by apache.

the class SimpleDescriptorHandlerTest method testInvalidServiceURLFromDiscovery.

/**
 * KNOX-1006
 *
 * Verify the behavior of the SimpleDescriptorHandler when service discovery fails to produce a valid URL for
 * a service.
 *
 * N.B. This test depends on the PropertiesFileServiceDiscovery extension being configured:
 *             org.apache.knox.gateway.topology.discovery.test.extension.PropertiesFileServiceDiscovery
 */
@Test
public void testInvalidServiceURLFromDiscovery() throws Exception {
    final String CLUSTER_NAME = "myproperties";
    // Configure the PropertiesFile Service Discovery implementation for this test
    final String DEFAULT_VALID_SERVICE_URL = "http://localhost:9999/thiswillwork";
    Properties serviceDiscoverySourceProps = new Properties();
    serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".NAMENODE", DEFAULT_VALID_SERVICE_URL.replace("http", "hdfs"));
    serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".JOBTRACKER", DEFAULT_VALID_SERVICE_URL.replace("http", "rpc"));
    serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".WEBHDFS", DEFAULT_VALID_SERVICE_URL);
    serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".WEBHCAT", DEFAULT_VALID_SERVICE_URL);
    serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".OOZIE", DEFAULT_VALID_SERVICE_URL);
    serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".WEBHBASE", DEFAULT_VALID_SERVICE_URL);
    serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".HIVE", "{SCHEME}://localhost:10000/");
    serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".RESOURCEMANAGER", DEFAULT_VALID_SERVICE_URL);
    serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".AMBARIUI", DEFAULT_VALID_SERVICE_URL);
    File serviceDiscoverySource = File.createTempFile("service-discovery", ".properties");
    serviceDiscoverySourceProps.store(new FileOutputStream(serviceDiscoverySource), "Test Service Discovery Source");
    // Prepare a mock SimpleDescriptor
    final String type = "PROPERTIES_FILE";
    final String address = serviceDiscoverySource.getAbsolutePath();
    final Map<String, List<String>> serviceURLs = new HashMap<>();
    serviceURLs.put("NAMENODE", null);
    serviceURLs.put("JOBTRACKER", null);
    serviceURLs.put("WEBHDFS", null);
    serviceURLs.put("WEBHCAT", null);
    serviceURLs.put("OOZIE", null);
    serviceURLs.put("WEBHBASE", null);
    serviceURLs.put("HIVE", null);
    serviceURLs.put("RESOURCEMANAGER", null);
    serviceURLs.put("AMBARIUI", Collections.singletonList("http://c6401.ambari.apache.org:8080"));
    // Write the externalized provider config to a temp file
    File providerConfig = writeProviderConfig("ambari-cluster-policy.xml", TEST_PROVIDER_CONFIG);
    File topologyFile = null;
    try {
        File destDir = (new File(".")).getCanonicalFile();
        GatewayConfig gc = EasyMock.createNiceMock(GatewayConfig.class);
        EasyMock.replay(gc);
        // Mock out the simple descriptor
        SimpleDescriptor testDescriptor = EasyMock.createNiceMock(SimpleDescriptor.class);
        EasyMock.expect(testDescriptor.getName()).andReturn("mysimpledescriptor").anyTimes();
        EasyMock.expect(testDescriptor.getDiscoveryAddress()).andReturn(address).anyTimes();
        EasyMock.expect(testDescriptor.getDiscoveryType()).andReturn(type).anyTimes();
        EasyMock.expect(testDescriptor.getDiscoveryUser()).andReturn(null).anyTimes();
        EasyMock.expect(testDescriptor.getProviderConfig()).andReturn(providerConfig.getAbsolutePath()).anyTimes();
        EasyMock.expect(testDescriptor.getClusterName()).andReturn(CLUSTER_NAME).anyTimes();
        List<SimpleDescriptor.Service> serviceMocks = new ArrayList<>();
        for (String serviceName : serviceURLs.keySet()) {
            SimpleDescriptor.Service svc = EasyMock.createNiceMock(SimpleDescriptor.Service.class);
            EasyMock.expect(svc.getName()).andReturn(serviceName).anyTimes();
            EasyMock.expect(svc.getURLs()).andReturn(serviceURLs.get(serviceName)).anyTimes();
            EasyMock.replay(svc);
            serviceMocks.add(svc);
        }
        EasyMock.expect(testDescriptor.getServices()).andReturn(serviceMocks).anyTimes();
        EasyMock.replay(testDescriptor);
        // Invoke the simple descriptor handler
        Map<String, File> files = SimpleDescriptorHandler.handle(gc, testDescriptor, // simple desc co-located with provider config
        providerConfig.getParentFile(), destDir);
        topologyFile = files.get("topology");
        // Validate the resulting topology descriptor
        assertTrue(topologyFile.exists());
        // Validate the topology descriptor's correctness
        TopologyValidator validator = new TopologyValidator(topologyFile.getAbsolutePath());
        if (!validator.validateTopology()) {
            throw new SAXException(validator.getErrorString());
        }
        XPathFactory xPathfactory = XPathFactory.newInstance();
        XPath xpath = xPathfactory.newXPath();
        // Parse the topology descriptor
        Document topologyXml = XmlUtils.readXml(topologyFile);
        // Validate the provider configuration
        Node gatewayNode = (Node) xpath.compile("/topology/gateway").evaluate(topologyXml, XPathConstants.NODE);
        ProviderConfiguration testProviderConfiguration = ProviderConfigurationParser.parseXML(new ByteArrayInputStream(TEST_PROVIDER_CONFIG.getBytes()));
        validateGeneratedProviderConfiguration(testProviderConfiguration, gatewayNode);
        // Validate the service declarations
        List<String> topologyServices = new ArrayList<>();
        Map<String, List<String>> topologyServiceURLs = new HashMap<>();
        NodeList serviceNodes = (NodeList) xpath.compile("/topology/service").evaluate(topologyXml, XPathConstants.NODESET);
        for (int serviceNodeIndex = 0; serviceNodeIndex < serviceNodes.getLength(); serviceNodeIndex++) {
            Node serviceNode = serviceNodes.item(serviceNodeIndex);
            Node roleNode = (Node) xpath.compile("role/text()").evaluate(serviceNode, XPathConstants.NODE);
            assertNotNull(roleNode);
            String role = roleNode.getNodeValue();
            topologyServices.add(role);
            NodeList urlNodes = (NodeList) xpath.compile("url/text()").evaluate(serviceNode, XPathConstants.NODESET);
            for (int urlNodeIndex = 0; urlNodeIndex < urlNodes.getLength(); urlNodeIndex++) {
                Node urlNode = urlNodes.item(urlNodeIndex);
                assertNotNull(urlNode);
                String url = urlNode.getNodeValue();
                assertNotNull("Every declared service should have a URL.", url);
                if (!topologyServiceURLs.containsKey(role)) {
                    topologyServiceURLs.put(role, new ArrayList<>());
                }
                topologyServiceURLs.get(role).add(url);
            }
        }
        // There should not be a service element for HIVE, since it had no valid URLs
        assertEquals("Unexpected number of service declarations.", serviceURLs.size() - 1, topologyServices.size());
        assertFalse("The HIVE service should have been omitted from the generated topology.", topologyServices.contains("HIVE"));
        assertEquals("Unexpected number of service URLs.", serviceURLs.size() - 1, topologyServiceURLs.size());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        serviceDiscoverySource.delete();
        providerConfig.delete();
        if (topologyFile != null) {
            topologyFile.delete();
        }
    }
}

Example 3 with TopologyValidator

use of org.apache.knox.gateway.topology.validation.TopologyValidator in project knox by apache.

the class DefaultTopologyService method redeployTopology.

private void redeployTopology(Topology topology) {
    File topologyFile = new File(topology.getUri());
    try {
        TopologyValidator tv = new TopologyValidator(topology);
        if (tv.validateTopology()) {
            throw new SAXException(tv.getErrorString());
        }
        long start = System.currentTimeMillis();
        // One second.
        long limit = 1000L;
        long elapsed = 1;
        while (elapsed <= limit) {
            try {
                long origTimestamp = topologyFile.lastModified();
                long setTimestamp = Math.max(System.currentTimeMillis(), topologyFile.lastModified() + elapsed);
                if (topologyFile.setLastModified(setTimestamp)) {
                    long newTimstamp = topologyFile.lastModified();
                    if (newTimstamp > origTimestamp) {
                        break;
                    } else {
                        Thread.sleep(10);
                        elapsed = System.currentTimeMillis() - start;
                        continue;
                    }
                } else {
                    auditor.audit(Action.REDEPLOY, topology.getName(), ResourceType.TOPOLOGY, ActionOutcome.FAILURE);
                    log.failedToRedeployTopology(topology.getName());
                    break;
                }
            } catch (InterruptedException e) {
                auditor.audit(Action.REDEPLOY, topology.getName(), ResourceType.TOPOLOGY, ActionOutcome.FAILURE);
                log.failedToRedeployTopology(topology.getName(), e);
                e.printStackTrace();
            }
        }
    } catch (SAXException e) {
        auditor.audit(Action.REDEPLOY, topology.getName(), ResourceType.TOPOLOGY, ActionOutcome.FAILURE);
        log.failedToRedeployTopology(topology.getName(), e);
    }
}

Example 4 with TopologyValidator

use of org.apache.knox.gateway.topology.validation.TopologyValidator in project knox by apache.

the class DefaultTopologyService method deployTopology.

public void deployTopology(Topology t) {
    try {
        File temp = new File(topologiesDirectory.getAbsolutePath() + "/" + t.getName() + ".xml.temp");
        Package topologyPkg = Topology.class.getPackage();
        String pkgName = topologyPkg.getName();
        String bindingFile = pkgName.replace(".", "/") + "/topology_binding-xml.xml";
        Map<String, Object> properties = new HashMap<>(1);
        properties.put(JAXBContextProperties.OXM_METADATA_SOURCE, bindingFile);
        JAXBContext jc = JAXBContext.newInstance(pkgName, Topology.class.getClassLoader(), properties);
        Marshaller mr = jc.createMarshaller();
        mr.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
        mr.marshal(t, temp);
        File topology = new File(topologiesDirectory.getAbsolutePath() + "/" + t.getName() + ".xml");
        if (!temp.renameTo(topology)) {
            FileUtils.forceDelete(temp);
            throw new IOException("Could not rename temp file");
        }
        // This code will check if the topology is valid, and retrieve the errors if it is not.
        TopologyValidator validator = new TopologyValidator(topology.getAbsolutePath());
        if (!validator.validateTopology()) {
            throw new SAXException(validator.getErrorString());
        }
    } catch (JAXBException e) {
        auditor.audit(Action.DEPLOY, t.getName(), ResourceType.TOPOLOGY, ActionOutcome.FAILURE);
        log.failedToDeployTopology(t.getName(), e);
    } catch (IOException io) {
        auditor.audit(Action.DEPLOY, t.getName(), ResourceType.TOPOLOGY, ActionOutcome.FAILURE);
        log.failedToDeployTopology(t.getName(), io);
    } catch (SAXException sx) {
        auditor.audit(Action.DEPLOY, t.getName(), ResourceType.TOPOLOGY, ActionOutcome.FAILURE);
        log.failedToDeployTopology(t.getName(), sx);
    }
    reloadTopologies();
}