Examples with ApplicationType com.linkedin.drelephant.analysis.ApplicationType used on opensource projects

Example 1 with ApplicationType

use of com.linkedin.drelephant.analysis.ApplicationType in project dr-elephant by linkedin.

the class Web method restSearchOptions.

/**
 * This returns the rest search options which are filled in the forms for the search page.
 * @return Returns the json object which should be filled in the search form.
 * return object:
 * <pre>
 *  *{
 *  "search-options": {
 *    "jobcategory": [
 *      {
 *        "name": "SPARK",
 *        "jobtypes": [
 *          {
 *            "name": "Spark"
 *          }
 *        ],
 *        "heuristics": [
 *          {
 *            "name": "Spark Configuration Best Practice"
 *          },
 *          {
 *            "name": "Spark Memory Limit"
 *          },
 *          {
 *            "name": "Spark Stage Runtime"
 *          },
 *          {
 *            "name": "Spark Job Runtime"
 *          },
 *          {
 *            "name": "Spark Executor Load Balance"
 *          },
 *          {
 *            "name": "Spark Event Log Limit"
 *          }
 *        ]
 *      },
 *      {
 *        "name": "MAPREDUCE",
 *        "jobtypes": [
 *          {
 *            "name": "Pig"
 *          },
 *          {
 *            "name": "Hive"
 *          },
 *          {
 *            "name": "Cascading"
 *          },
 *          {
 *            "name": "Voldemort"
 *          },
 *          {
 *            "name": "Kafka"
 *          },
 *          {
 *            "name": "HadoopJava"
 *          }
 *        ],
 *        "heuristics": [
 *          {
 *            "name": "Mapper Data Skew"
 *          },
 *          {
 *            "name": "Mapper GC"
 *          },
 *          {
 *            "name": "Mapper Time"
 *          },
 *          {
 *            "name": "Mapper Speed"
 *          },
 *          {
 *            "name": "Mapper Spill"
 *          },
 *          {
 *            "name": "Mapper Memory"
 *          },
 *          {
 *            "name": "Reducer Data Skew"
 *          },
 *          {
 *            "name": "Reducer GC"
 *          },
 *          {
 *            "name": "Reducer Time"
 *          },
 *          {
 *            "name": "Reducer Memory"
 *          },
 *          {
 *            "name": "Shuffle & Sort"
 *          },
 *          {
 *            "name": "Exception"
 *          }
 *        ]
 *      }
 *    ],
 *    "severities": [
 *      {
 *        "name": "Critical",
 *        "value": 4
 *      },
 *      {
 *        "name": "Severe",
 *        "value": 3
 *      },
 *      {
 *        "name": "Moderate",
 *        "value": 2
 *      },
 *      {
 *        "name": "Low",
 *        "value": 1
 *      },
 *      {
 *        "name": "None",
 *        "value": 0
 *      }
 *    ],
 *    "id": "search"
 *  }
 *}
 * </pre>
 */
public static Result restSearchOptions() {
    JsonObject searchOptions = new JsonObject();
    JsonArray jobCategory = new JsonArray();
    JsonArray severities = new JsonArray();
    Map<ApplicationType, List<JobType>> applicationTypeListMap = ElephantContext.instance().getAppTypeToJobTypes();
    for (ApplicationType key : applicationTypeListMap.keySet()) {
        JsonObject applicationType = new JsonObject();
        JsonArray jobTypes = new JsonArray();
        JsonArray heuristics = new JsonArray();
        for (JobType jobtype : applicationTypeListMap.get(key)) {
            JsonObject jobTypeNode = new JsonObject();
            jobTypeNode.addProperty(JsonKeys.NAME, jobtype.getName());
            jobTypes.add(jobTypeNode);
        }
        for (Heuristic heuristic : ElephantContext.instance().getHeuristicsForApplicationType(key)) {
            JsonObject heuristicNode = new JsonObject();
            heuristicNode.addProperty(JsonKeys.NAME, heuristic.getHeuristicConfData().getHeuristicName());
            heuristics.add(heuristicNode);
        }
        applicationType.addProperty(JsonKeys.NAME, key.getName());
        applicationType.add(JsonKeys.JOB_TYPES, jobTypes);
        applicationType.add(JsonKeys.HEURISTICS, heuristics);
        jobCategory.add(applicationType);
    }
    for (Severity severity : Severity.values()) {
        JsonObject severityObject = new JsonObject();
        severityObject.addProperty(JsonKeys.NAME, severity.getText());
        severityObject.addProperty(JsonKeys.VALUE, severity.getValue());
        severities.add(severityObject);
    }
    searchOptions.add(JsonKeys.JOB_CATEGORY, jobCategory);
    searchOptions.add(JsonKeys.SEVERITIES, severities);
    searchOptions.addProperty(JsonKeys.ID, "search");
    JsonObject parent = new JsonObject();
    parent.add(JsonKeys.SEARCH_OPTS, searchOptions);
    return ok(new Gson().toJson(parent));
}

Example 2 with ApplicationType

use of com.linkedin.drelephant.analysis.ApplicationType in project dr-elephant by linkedin.

the class ElephantContext method configureSupportedApplicationTypes.

/**
 * Decides what application types can be supported.
 *
 * An application type is supported if all the below are true.
 * 1. A Fetcher is defined in FetcherConf.xml for the application type.
 * 2. At least one Heuristic is configured in HeuristicConf.xml for the application type.
 * 3. At least one job type is configured in JobTypeConf.xml for the application type.
 */
private void configureSupportedApplicationTypes() {
    Set<ApplicationType> supportedTypes = Sets.intersection(_typeToFetcher.keySet(), _typeToHeuristics.keySet());
    supportedTypes = Sets.intersection(supportedTypes, _appTypeToJobTypes.keySet());
    supportedTypes = Sets.intersection(supportedTypes, _typeToAggregator.keySet());
    _typeToAggregator.keySet().retainAll(supportedTypes);
    _typeToFetcher.keySet().retainAll(supportedTypes);
    _typeToHeuristics.keySet().retainAll(supportedTypes);
    _appTypeToJobTypes.keySet().retainAll(supportedTypes);
    logger.info("Configuring ElephantContext...");
    for (ApplicationType type : supportedTypes) {
        _nameToType.put(type.getName(), type);
        List<String> classes = new ArrayList<String>();
        List<Heuristic> heuristics = _typeToHeuristics.get(type);
        for (Heuristic heuristic : heuristics) {
            classes.add(heuristic.getClass().getName());
        }
        List<JobType> jobTypes = _appTypeToJobTypes.get(type);
        logger.info("Supports " + type.getName() + " application type, using " + _typeToFetcher.get(type).toString() + " fetcher class with Heuristics [" + StringUtils.join(classes, ", ") + "] and following JobTypes [" + StringUtils.join(jobTypes, ", ") + "].");
    }
}

Example 3 with ApplicationType

use of com.linkedin.drelephant.analysis.ApplicationType in project dr-elephant by linkedin.

the class HeuristicConfiguration method parseHeuristicConfiguration.

private void parseHeuristicConfiguration(Element configuration) {
    _heuristicsConfDataList = new ArrayList<HeuristicConfigurationData>();
    NodeList nodes = configuration.getChildNodes();
    int n = 0;
    for (int i = 0; i < nodes.getLength(); i++) {
        // Each heuristic node
        Node node = nodes.item(i);
        if (node.getNodeType() == Node.ELEMENT_NODE) {
            n++;
            Element heuristicElem = (Element) node;
            String className;
            Node classNameNode = heuristicElem.getElementsByTagName("classname").item(0);
            if (classNameNode == null) {
                throw new RuntimeException("No tag 'classname' in heuristic " + n);
            }
            className = classNameNode.getTextContent();
            if (className.equals("")) {
                throw new RuntimeException("Empty tag 'classname' in heuristic " + n);
            }
            String heuristicName;
            Node heuristicNameNode = heuristicElem.getElementsByTagName("heuristicname").item(0);
            if (heuristicNameNode == null) {
                throw new RuntimeException("No tag 'heuristicname' in heuristic " + n + " classname " + className);
            }
            heuristicName = heuristicNameNode.getTextContent();
            if (heuristicName.equals("")) {
                throw new RuntimeException("Empty tag 'heuristicname' in heuristic " + n + " classname " + className);
            }
            String viewName;
            Node viewNameNode = heuristicElem.getElementsByTagName("viewname").item(0);
            if (viewNameNode == null) {
                throw new RuntimeException("No tag 'viewname' in heuristic " + n + " classname " + className);
            }
            viewName = viewNameNode.getTextContent();
            if (viewName.equals("")) {
                throw new RuntimeException("Empty tag 'viewname' in heuristic " + n + " classname " + className);
            }
            Node appTypeNode = heuristicElem.getElementsByTagName("applicationtype").item(0);
            if (appTypeNode == null) {
                throw new RuntimeException("No tag or invalid tag 'applicationtype' in heuristic " + n + " classname " + className);
            }
            String appTypeStr = appTypeNode.getTextContent();
            if (appTypeStr == null) {
                logger.error("Application type is not specified in heuristic " + n + " classname " + className + ". Skipping this configuration.");
                continue;
            }
            ApplicationType appType = new ApplicationType(appTypeStr);
            // Check if parameters are defined for the heuristic
            Map<String, String> paramsMap = Utils.getConfigurationParameters(heuristicElem);
            HeuristicConfigurationData heuristicData = new HeuristicConfigurationData(heuristicName, className, viewName, appType, paramsMap);
            _heuristicsConfDataList.add(heuristicData);
        }
    }
}

Example 4 with ApplicationType

use of com.linkedin.drelephant.analysis.ApplicationType in project dr-elephant by linkedin.

the class ElephantContext method loadFetchers.

/**
 * Load all the fetchers configured in FetcherConf.xml
 */
private void loadFetchers() {
    Document document = Utils.loadXMLDoc(FETCHERS_CONF);
    _fetchersConfData = new FetcherConfiguration(document.getDocumentElement()).getFetchersConfigurationData();
    for (FetcherConfigurationData data : _fetchersConfData) {
        try {
            Class<?> fetcherClass = Class.forName(data.getClassName());
            Object instance = fetcherClass.getConstructor(FetcherConfigurationData.class).newInstance(data);
            if (!(instance instanceof ElephantFetcher)) {
                throw new IllegalArgumentException("Class " + fetcherClass.getName() + " is not an implementation of " + ElephantFetcher.class.getName());
            }
            ApplicationType type = data.getAppType();
            if (_typeToFetcher.get(type) == null) {
                _typeToFetcher.put(type, (ElephantFetcher) instance);
            }
            logger.info("Load Fetcher : " + data.getClassName());
        } catch (ClassNotFoundException e) {
            throw new RuntimeException("Could not find class " + data.getClassName(), e);
        } catch (InstantiationException e) {
            throw new RuntimeException("Could not instantiate class " + data.getClassName(), e);
        } catch (IllegalAccessException e) {
            throw new RuntimeException("Could not access constructor for class" + data.getClassName(), e);
        } catch (RuntimeException e) {
            throw new RuntimeException(data.getClassName() + " is not a valid Fetcher class.", e);
        } catch (InvocationTargetException e) {
            throw new RuntimeException("Could not invoke class " + data.getClassName(), e);
        } catch (NoSuchMethodException e) {
            throw new RuntimeException("Could not find constructor for class " + data.getClassName(), e);
        }
    }
}

Example 5 with ApplicationType

use of com.linkedin.drelephant.analysis.ApplicationType in project dr-elephant by linkedin.

the class ElephantContext method getAllHeuristicNames.

/**
 * Return the heuristic names available grouped by application type.
 *
 * @return A map of application type name -> a list of heuristic names
 */
public Map<String, List<String>> getAllHeuristicNames() {
    if (_heuristicGroupedNames.isEmpty()) {
        for (Map.Entry<ApplicationType, List<Heuristic>> entry : _typeToHeuristics.entrySet()) {
            ApplicationType type = entry.getKey();
            List<Heuristic> list = entry.getValue();
            List<String> nameList = new ArrayList<String>();
            for (Heuristic heuristic : list) {
                nameList.add(heuristic.getHeuristicConfData().getHeuristicName());
            }
            Collections.sort(nameList);
            _heuristicGroupedNames.put(type.getName(), nameList);
        }
    }
    return _heuristicGroupedNames;
}