Examples with ApplicationType com.linkedin.drelephant.analysis.ApplicationType used on opensource projects

Example 6 with ApplicationType

use of com.linkedin.drelephant.analysis.ApplicationType in project dr-elephant by linkedin.

the class ElephantContext method loadHeuristics.

/**
 * Load all the heuristics and their views configured in HeuristicConf.xml
 */
private void loadHeuristics() {
    Document document = Utils.loadXMLDoc(HEURISTICS_CONF);
    _heuristicsConfData = new HeuristicConfiguration(document.getDocumentElement()).getHeuristicsConfigurationData();
    for (HeuristicConfigurationData data : _heuristicsConfData) {
        // Load all the heuristic classes
        try {
            Class<?> heuristicClass = Class.forName(data.getClassName());
            Object instance = heuristicClass.getConstructor(HeuristicConfigurationData.class).newInstance(data);
            if (!(instance instanceof Heuristic)) {
                throw new IllegalArgumentException("Class " + heuristicClass.getName() + " is not an implementation of " + Heuristic.class.getName());
            }
            ApplicationType type = data.getAppType();
            List<Heuristic> heuristics = _typeToHeuristics.get(type);
            if (heuristics == null) {
                heuristics = new ArrayList<Heuristic>();
                _typeToHeuristics.put(type, heuristics);
            }
            heuristics.add((Heuristic) instance);
            logger.info("Load Heuristic : " + data.getClassName());
        } catch (ClassNotFoundException e) {
            throw new RuntimeException("Could not find class " + data.getClassName(), e);
        } catch (InstantiationException e) {
            throw new RuntimeException("Could not instantiate class " + data.getClassName(), e);
        } catch (IllegalAccessException e) {
            throw new RuntimeException("Could not access constructor for class" + data.getClassName(), e);
        } catch (RuntimeException e) {
            // More descriptive on other runtime exception such as ClassCastException
            throw new RuntimeException(data.getClassName() + " is not a valid Heuristic class.", e);
        } catch (InvocationTargetException e) {
            throw new RuntimeException("Could not invoke class " + data.getClassName(), e);
        } catch (NoSuchMethodException e) {
            throw new RuntimeException("Could not find constructor for class " + data.getClassName(), e);
        }
        // Load all the heuristic views
        try {
            Class<?> viewClass = Class.forName(data.getViewName());
            Method render = viewClass.getDeclaredMethod("render");
            Html page = (Html) render.invoke(null);
            _heuristicToView.put(data.getHeuristicName(), page);
            logger.info("Load View : " + data.getViewName());
        } catch (ClassNotFoundException e) {
            throw new RuntimeException("Could not find view " + data.getViewName(), e);
        } catch (IllegalAccessException e) {
            throw new RuntimeException("Could not access render on view" + data.getViewName(), e);
        } catch (RuntimeException e) {
            // More descriptive on other runtime exception such as ClassCastException
            throw new RuntimeException(data.getViewName() + " is not a valid view class.", e);
        } catch (InvocationTargetException e) {
            throw new RuntimeException("Could not invoke view " + data.getViewName(), e);
        } catch (NoSuchMethodException e) {
            throw new RuntimeException("Could not find method render for view " + data.getViewName(), e);
        }
    }
    // Bind No_DATA heuristic to its helper pages, no need to add any real configurations
    _heuristicsConfData.add(new HeuristicConfigurationData(HeuristicResult.NO_DATA.getHeuristicName(), HeuristicResult.NO_DATA.getHeuristicClassName(), "views.html.help.helpNoData", null, null));
}

Example 7 with ApplicationType

use of com.linkedin.drelephant.analysis.ApplicationType in project dr-elephant by linkedin.

the class ElephantContext method loadAggregators.

private void loadAggregators() {
    Document document = Utils.loadXMLDoc(AGGREGATORS_CONF);
    _aggregatorConfData = new AggregatorConfiguration(document.getDocumentElement()).getAggregatorsConfigurationData();
    for (AggregatorConfigurationData data : _aggregatorConfData) {
        try {
            Class<?> aggregatorClass = Class.forName(data.getClassName());
            Object instance = aggregatorClass.getConstructor(AggregatorConfigurationData.class).newInstance(data);
            if (!(instance instanceof HadoopMetricsAggregator)) {
                throw new IllegalArgumentException("Class " + aggregatorClass.getName() + " is not an implementation of " + HadoopMetricsAggregator.class.getName());
            }
            ApplicationType type = data.getAppType();
            if (_typeToAggregator.get(type) == null) {
                _typeToAggregator.put(type, (HadoopMetricsAggregator) instance);
            }
            logger.info("Load Aggregator : " + data.getClassName());
        } catch (ClassNotFoundException e) {
            throw new RuntimeException("Could not find class " + data.getClassName(), e);
        } catch (InstantiationException e) {
            throw new RuntimeException("Could not instantiate class " + data.getClassName(), e);
        } catch (IllegalAccessException e) {
            throw new RuntimeException("Could not access constructor for class" + data.getClassName(), e);
        } catch (RuntimeException e) {
            throw new RuntimeException(data.getClassName() + " is not a valid Aggregator class.", e);
        } catch (InvocationTargetException e) {
            throw new RuntimeException("Could not invoke class " + data.getClassName(), e);
        } catch (NoSuchMethodException e) {
            throw new RuntimeException("Could not find constructor for class " + data.getClassName(), e);
        }
    }
}

Example 8 with ApplicationType

use of com.linkedin.drelephant.analysis.ApplicationType in project dr-elephant by linkedin.

the class InfoExtractorTest method testLoadSchedulerInfo.

@Test
public void testLoadSchedulerInfo() {
    Properties properties = new Properties();
    properties.put(AzkabanScheduler.AZKABAN_JOB_URL, "https://grid.example.com:9000/manager?project=project-name&flow=flow-name&job=job-name");
    properties.put(AzkabanScheduler.AZKABAN_ATTEMPT_URL, "https://grid.example.com:9000/executor?execid=123456&job=job-name&attempt=0");
    properties.put(AzkabanScheduler.AZKABAN_WORKFLOW_URL, "https://grid.example.com:9000/manager?project=project-name&flow=flow-name");
    properties.put(AzkabanScheduler.AZKABAN_EXECUTION_URL, "https://grid.example.com:9000/executor?execid=123456");
    properties.put(AzkabanScheduler.AZKABAN_JOB_NAME, "job-name");
    SchedulerConfigurationData schedulerConfigurationData = new SchedulerConfigurationData("azkaban", null, null);
    Scheduler scheduler = new AzkabanScheduler("id", properties, schedulerConfigurationData);
    AppResult result = new AppResult();
    HadoopApplicationData data = new HadoopApplicationData() {

        String appId = "application_5678";

        Properties conf = new Properties();

        ApplicationType applicationType = new ApplicationType("foo");

        @Override
        public String getAppId() {
            return appId;
        }

        @Override
        public Properties getConf() {
            return conf;
        }

        @Override
        public ApplicationType getApplicationType() {
            return applicationType;
        }

        @Override
        public boolean isEmpty() {
            return false;
        }
    };
    InfoExtractor.loadSchedulerInfo(result, data, scheduler);
    assertEquals(result.scheduler, "azkaban");
    assertFalse(StringUtils.isEmpty(result.getJobExecId()));
    assertFalse(StringUtils.isEmpty(result.getJobDefId()));
    assertFalse(StringUtils.isEmpty(result.getFlowExecId()));
    assertFalse(StringUtils.isEmpty(result.getFlowDefId()));
    assertFalse(StringUtils.isEmpty(result.getJobExecUrl()));
    assertFalse(StringUtils.isEmpty(result.getJobDefUrl()));
    assertFalse(StringUtils.isEmpty(result.getFlowExecUrl()));
    assertFalse(StringUtils.isEmpty(result.getFlowDefUrl()));
}

Example 9 with ApplicationType

use of com.linkedin.drelephant.analysis.ApplicationType in project dr-elephant by linkedin.

the class AggregatorConfiguration method parseAggregatorConfiguration.

/**
 * Parses the Aggregator configuration file and loads the Aggregator Information to a list of AggregatorConfigurationData
 *
 * @param configuration The dom Element to be parsed
 */
private void parseAggregatorConfiguration(Element configuration) {
    _aggregatorsConfDataList = new ArrayList<AggregatorConfigurationData>();
    NodeList nodes = configuration.getChildNodes();
    int n = 0;
    for (int i = 0; i < nodes.getLength(); i++) {
        Node node = nodes.item(i);
        if (node.getNodeType() == Node.ELEMENT_NODE) {
            n++;
            Element aggregatorNode = (Element) node;
            String className;
            Node classNameNode = aggregatorNode.getElementsByTagName("classname").item(0);
            if (classNameNode == null) {
                throw new RuntimeException("No tag 'classname' in aggregator " + n);
            }
            className = classNameNode.getTextContent();
            if (className.equals("")) {
                throw new RuntimeException("Empty tag 'classname' in aggregator " + n);
            }
            Node appTypeNode = aggregatorNode.getElementsByTagName("applicationtype").item(0);
            if (appTypeNode == null) {
                throw new RuntimeException("No tag or invalid tag 'applicationtype' in aggregator " + n + " classname " + className);
            }
            String appTypeStr = appTypeNode.getTextContent();
            if (appTypeStr == null) {
                logger.error("Application type is not specified in aggregator " + n + " classname " + className + ". Skipping this configuration.");
                continue;
            }
            ApplicationType appType = new ApplicationType(appTypeStr);
            // Check if parameters are defined for the heuristic
            Map<String, String> paramsMap = Utils.getConfigurationParameters(aggregatorNode);
            AggregatorConfigurationData aggregatorData = new AggregatorConfigurationData(className, appType, paramsMap);
            _aggregatorsConfDataList.add(aggregatorData);
        }
    }
}

Example 10 with ApplicationType

use of com.linkedin.drelephant.analysis.ApplicationType in project dr-elephant by linkedin.

the class FetcherConfiguration method parseFetcherConfiguration.

/**
 * Parses the Fetcher configuration file and loads the Fetcher Information to a list of FetcherConfigurationData
 *
 * @param configuration The dom Element to be parsed
 */
private void parseFetcherConfiguration(Element configuration) {
    _fetchersConfDataList = new ArrayList<FetcherConfigurationData>();
    NodeList nodes = configuration.getChildNodes();
    int n = 0;
    for (int i = 0; i < nodes.getLength(); i++) {
        // Each heuristic node
        Node node = nodes.item(i);
        if (node.getNodeType() == Node.ELEMENT_NODE) {
            n++;
            Element fetcherElem = (Element) node;
            String className;
            Node classNameNode = fetcherElem.getElementsByTagName("classname").item(0);
            if (classNameNode == null) {
                throw new RuntimeException("No tag 'classname' in fetcher " + n);
            }
            className = classNameNode.getTextContent();
            if (className.equals("")) {
                throw new RuntimeException("Empty tag 'classname' in fetcher " + n);
            }
            Node appTypeNode = fetcherElem.getElementsByTagName("applicationtype").item(0);
            if (appTypeNode == null) {
                throw new RuntimeException("No tag or invalid tag 'applicationtype' in fetcher " + n + " classname " + className);
            }
            String appTypeStr = appTypeNode.getTextContent();
            if (appTypeStr == null) {
                logger.error("Application type is not specified in fetcher " + n + " classname " + className + ". Skipping this configuration.");
                continue;
            }
            ApplicationType appType = new ApplicationType(appTypeStr);
            // Check if parameters are defined for the heuristic
            Map<String, String> paramsMap = Utils.getConfigurationParameters(fetcherElem);
            FetcherConfigurationData fetcherData = new FetcherConfigurationData(className, appType, paramsMap);
            _fetchersConfDataList.add(fetcherData);
        }
    }
}