Examples with UserDefinedMetrics com.yahoo.dba.perf.myperf.common.UserDefinedMetrics used on opensource projects

Example 1 with UserDefinedMetrics

use of com.yahoo.dba.perf.myperf.common.UserDefinedMetrics in project mysql_perf_analyzer by yahoo.

the class MetricsDbBase method setMetricsGroups.

/**
	   * This is used during startup
	   * @param mgs
	   */
public void setMetricsGroups(MetricsDefManager metricsDef) {
    synchronized (metricsDefLock) {
        String[] groupNames = metricsDef.getGroupNames();
        for (String grpName : groupNames) {
            MetricsGroup group = metricsDef.getGroupByName(grpName);
            //not supposed to be so
            if (group == null)
                continue;
            List<MetricsGroup> mgs = new ArrayList<MetricsGroup>();
            if (//no sub group, add self
            group.getSubGroups().size() == 0) {
                mgs.add(group);
            } else {
                for (MetricsGroup g : group.getSubGroups()) mgs.add(g);
            }
            for (MetricsGroup g : mgs) {
                String sinkName = g.getSinkTableName();
                this.metricsGroups.put(sinkName, g);
                //2014-02-14, change size to 10K
                this.dataQueues.put(sinkName, new java.util.concurrent.ArrayBlockingQueue<MetricsData>(10000));
                if (!g.isStoreInCommonTable())
                    this.insertSQL.put(sinkName, this.insertSQL(g));
                else {
                    String targetTable = g.getTargetTable();
                    if (targetTable == null || targetTable.isEmpty())
                        targetTable = "METRIC_GENERIC";
                    this.insertSQL.put((g.getDbType() + "_" + g.getGroupName()), "INSERT INTO " + targetTable + " (DBID, METRIC_ID, SNAP_ID, TS, VALUE) VALUES(?,?,?,?,?)");
                }
            }
        }
        for (Map.Entry<String, UserDefinedMetrics> entry : metricsDef.getUdmManager().getUdms().entrySet()) {
            MetricsGroup group = entry.getValue().getMetricsGroup();
            //not supposed to be so
            if (group == null)
                continue;
            String sinkName = group.getSinkTableName();
            this.metricsGroups.put(sinkName, group);
            //2014-02-14, change size to 10K	      
            this.dataQueues.put(sinkName, new java.util.concurrent.ArrayBlockingQueue<MetricsData>(10000));
            this.insertSQL.put(sinkName, this.insertSQL(group));
        }
    }
}

Example 2 with UserDefinedMetrics

use of com.yahoo.dba.perf.myperf.common.UserDefinedMetrics in project mysql_perf_analyzer by yahoo.

the class MetricsRetentionTask method run.

@Override
public void run() {
    Thread.currentThread().setName("MetricsRetentionTask");
    logger.info("Starting metrics purge job");
    if (this.context.getMetricDb() == null) {
        logger.info("MetricsDB has yet to set.");
        return;
    }
    //get all dbids
    List<Integer> ids = new ArrayList<Integer>();
    if (this.dbidToPurge == null) {
        for (Map.Entry<String, DBGroupInfo> e : context.getDbInfoManager().getClusters().entrySet()) {
            DBGroupInfo g = e.getValue();
            for (DBInstanceInfo i : g.getInstances()) {
                ids.add(i.getDbid());
            }
        }
    } else {
        for (int id : this.dbidToPurge) ids.add(id);
    }
    Calendar c = Calendar.getInstance();
    c.add(Calendar.DATE, -this.renentionDays);
    Date dt = c.getTime();
    //now get current timestamp
    java.text.SimpleDateFormat sdf = new java.text.SimpleDateFormat("yyyyMMddHHmmss");
    sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
    long endDate = Long.parseLong(sdf.format(dt));
    List<MetricsGroup> mgs = new ArrayList<MetricsGroup>();
    String[] groupNames = this.context.getMetricsDef().getGroupNames();
    for (String grpName : groupNames) {
        MetricsGroup group = this.context.getMetricsDef().getGroupByName(grpName);
        //not supposed to be so
        if (group == null)
            continue;
        if (//no sub group, add self
        group.getSubGroups().size() == 0) {
            mgs.add(group);
        } else {
            for (MetricsGroup g : group.getSubGroups()) mgs.add(g);
        }
    }
    for (Map.Entry<String, UserDefinedMetrics> entry : this.context.getMetricsDef().getUdmManager().getUdms().entrySet()) {
        MetricsGroup group = entry.getValue().getMetricsGroup();
        //not supposed to be so
        if (group == null)
            continue;
        mgs.add(group);
    }
    for (int dbid : ids) {
        logger.info("Check and purge db: " + dbid);
        for (MetricsGroup g : mgs) {
            if (this.dbidToPurge != null)
                this.context.getMetricDb().purgeAll(g.getSinkTableName(), dbid);
            else
                this.context.getMetricDb().purge(g.getSinkTableName(), dbid, endDate);
        }
    }
    java.text.SimpleDateFormat sdf2 = new java.text.SimpleDateFormat("yyyyMMdd");
    sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
    purgeAlertReports(Integer.parseInt(sdf2.format(dt)));
    logger.info("Ended metrics purge job");
    if (this.dbidToPurge == null)
        this.context.getMetricDb().purgeAlerts(endDate);
    //for now, we only do it once a day.
    //TODO keep consistency with DB add/update/delete
    logger.info("Retention job done.");
}

Example 3 with UserDefinedMetrics

use of com.yahoo.dba.perf.myperf.common.UserDefinedMetrics in project mysql_perf_analyzer by yahoo.

the class UdmController method handleRequestImpl.

@Override
protected ModelAndView handleRequestImpl(HttpServletRequest req, HttpServletResponse resp) throws Exception {
    String message = "OK";
    String cmd = req.getParameter("cmd");
    boolean isValid = true;
    UserDefinedMetrics udm = null;
    logger.info("receive cmd " + cmd + ", url " + req.getQueryString());
    //it a form request
    if ((cmd == null || cmd.isEmpty()) && !WebAppUtil.hasValidSession(req)) {
        return new ModelAndView(new RedirectView(nosessView));
    }
    //first session check
    boolean isSessionValid = WebAppUtil.hasValidSession(req);
    if (!isSessionValid)
        return this.respondFailure("session timeout. Please logout and re-login.", req);
    AppUser appUser = AppUser.class.cast(req.getSession().getAttribute(AppUser.SESSION_ATTRIBUTE));
    ModelAndView mv = null;
    //By default, just display the page
    if (//or any invalid command
    cmd == null || cmd.isEmpty())
        message = null;
    if ("test".equals(cmd) || "publish".equals(cmd)) {
        if (appUser.isAdminUser())
            return this.processNewUDM(req, resp);
        else
            return this.respondFailure("Non admin user is not allowed to add UDM", req);
    } else if ("test_alert".equals(cmd) || "publish_alert".equals(cmd)) {
        if (appUser.isAdminUser())
            return this.processNewAlert(req, resp);
        else
            return this.respondFailure("Non admin user is not allowed to add UDM", req);
    } else if (//display
    "udm_detail".equalsIgnoreCase(cmd)) {
        String name = req.getParameter("name");
        udm = this.frameworkContext.getMetricsDef().getUdmManager().getUDMByName(name);
        if (udm != null) {
            mv = new ModelAndView(this.jsonView);
            String res = "{\"resp\":{\"status\": 0, \"message\":\"OK\", \"udm\": " + udm.toJSON(true) + "}}";
            //logger.info("UDM detail: "+res);
            mv.addObject("json_result", res);
            return mv;
        } else
            return this.respondFailure("UDM " + name + " not found", req);
    } else if (//display
    "alert_detail".equalsIgnoreCase(cmd)) {
        String name = req.getParameter("name");
        AlertDefinition alert = this.frameworkContext.getMetricsDef().getUdmManager().getAlertByName(name);
        if (alert != null) {
            mv = new ModelAndView(this.jsonView);
            String res = "{\"resp\":{\"status\": 0, \"message\":\"OK\", \"alert\": " + alert.toJSON(true) + "}}";
            //logger.info("UDM detail: "+res);
            mv.addObject("json_result", res);
            return mv;
        } else
            return this.respondFailure("ALERT " + name + " not found", req);
    } else if (//display
    "udmdb_detail".equalsIgnoreCase(cmd))
        return processUDMDBDetail(req, resp);
    else if (//display
    "udmdb_update".equalsIgnoreCase(cmd))
        return processUDMDBUpdate(req, resp, appUser);
    else if (//display
    "alertdb_update".equalsIgnoreCase(cmd))
        return this.processAlertDBUpdate(req, resp, appUser);
    mv = new ModelAndView(this.formView);
    //udm list
    List<String> udms = new ArrayList<String>();
    for (Map.Entry<String, UserDefinedMetrics> e : this.frameworkContext.getMetricsDef().getUdmManager().getUdms().entrySet()) {
        udms.add(e.getKey());
    }
    mv.addObject("udms", udms);
    //alerts
    List<String> alerts = new ArrayList<String>();
    for (Map.Entry<String, AlertDefinition> e : this.frameworkContext.getMetricsDef().getUdmManager().getAlerts().entrySet()) {
        alerts.add(e.getKey());
    }
    mv.addObject("alerts", alerts);
    //predefined
    Map<String, String> predefined = new TreeMap<String, String>();
    String[] predefinedGroups = this.frameworkContext.getMetricsDef().getGroupNames();
    for (String gname : predefinedGroups) {
        MetricsGroup g = this.frameworkContext.getMetricsDef().getGroupByName(gname);
        List<MetricsGroup> subGroups = g.getSubGroups();
        if (subGroups != null && subGroups.size() > 0) {
            for (MetricsGroup subG : subGroups) predefined.put(gname + "." + subG.getGroupName(), subG.isAuto() ? "y" : "n");
        } else
            predefined.put(gname, g.isAuto() ? "y" : "n");
    }
    mv.addObject("predefined", predefined);
    if (WebAppUtil.hasValidSession(req)) {
        mv.addObject("mydbs", this.frameworkContext.getDbInfoManager().listDbsByUserInfo(WebAppUtil.findUserFromRequest(req), retrieveAppUser(req).isRestrictedUser()));
        mv.addObject("mydbSize", this.frameworkContext.getDbInfoManager().getMyDatabases(WebAppUtil.findUserFromRequest(req), retrieveAppUser(req).isRestrictedUser()).size());
    } else {
        mv.addObject("mydbs", this.frameworkContext.getDbInfoManager().getClusters().keySet());
        mv.addObject("mydbSize", 0);
    }
    mv.addObject("dbMap", this.frameworkContext.getDbInfoManager().getClusters());
    mv.addObject("help_key", "udm");
    if (!isValid && message != null)
        mv.addObject("message", message);
    else if (message != null)
        mv.addObject("okmessage", message);
    if (cmd != null)
        mv.addObject("cmd", cmd);
    mv.addObject("u", appUser);
    return mv;
}

Example 4 with UserDefinedMetrics

use of com.yahoo.dba.perf.myperf.common.UserDefinedMetrics in project mysql_perf_analyzer by yahoo.

the class UdmController method testAndValidateAlert.

private void testAndValidateAlert(HttpServletRequest req, AlertDefinition def, String db) throws Exception {
    if (def == null)
        throw new Exception("please provide valid alert definition for test");
    if (AlertDefinition.SOURCE_SQL.equals(def.getSource())) {
        if (def.getSqlText() == null || def.getSqlText().isEmpty())
            throw new Exception("please provide valid SQL text for test");
        if (db == null || db.isEmpty())
            throw new Exception("please provide valid database for test");
        String[] dbs = db.split("\\|");
        if (dbs == null || dbs.length < 2)
            throw new Exception("please provide valid database for test");
        DBInstanceInfo dbinfo = this.frameworkContext.getDbInfoManager().findDB(dbs[0], dbs[1]);
        if (dbinfo == null)
            throw new Exception("please provide valid database for test");
        DBConnectionWrapper connWrapper = null;
        Statement stmt = null;
        ResultSet rs = null;
        try {
            connWrapper = WebAppUtil.getDBConnection(req, this.frameworkContext, dbinfo);
            if (connWrapper == null) {
                throw new Exception("failed to connect to target db (" + dbinfo + ")");
            }
            String sql = def.getSqlText();
            if (def.getParams() != null && def.getParams().size() > 0) {
                for (Map.Entry<String, String> e : def.getParams().entrySet()) {
                    sql = sql.replace("&" + e.getKey(), e.getValue());
                }
            }
            stmt = connWrapper.getConnection().createStatement();
            rs = stmt.executeQuery(sql);
        } finally {
            DBUtils.close(rs);
            DBUtils.close(stmt);
            //close it anyway
            WebAppUtil.closeDBConnection(req, connWrapper, true, false);
        }
    } else if (AlertDefinition.SOURCE_METRICS.equals(def.getSource())) {
        String metricsName = def.getMetricName();
        if (metricsName == null || metricsName.isEmpty())
            throw new Exception("please provide valid metrics name in the format of group[.subgroup].metric_name");
        String[] ms = metricsName.split("\\.");
        if (ms.length < 2)
            throw new Exception("please provide valid metrics name in the format of group[.subgroup].metric_name. " + metricsName + " is not valid.");
        MetricsGroup mg = this.frameworkContext.getMetricsDef().getGroupByName(ms[0]);
        if (mg == null) {
            //try UDM
            UserDefinedMetrics udm = this.frameworkContext.getMetricsDef().getUdmManager().getUDMByName(ms[0]);
            if (udm == null)
                throw new Exception("please provide valid metrics name in the format of group[.subgroup].metric_name. " + ms[0] + " is not valid metrics group name.");
            else
                mg = udm.getMetricsGroup();
        }
        if (mg == null)
            throw new Exception("please provide valid metrics name in the format of group[.subgroup].metric_name. " + ms[0] + " is not valid metrics group name.");
        if (mg.getSubGroups().size() > 0) {
            if (ms.length < 3)
                throw new Exception("please provide valid metrics name in the format of group[.subgroup].metric_name. " + ms[0] + " has sub groups.");
            else
                mg = mg.getSubGroupByName(ms[1]);
            if (mg == null)
                throw new Exception("please provide valid metrics name in the format of group[.subgroup].metric_name. " + ms[1] + " is not a valid sub group in group " + ms[0]);
        }
        boolean findOne = false;
        for (Metric m : mg.getMetrics()) {
            if (m.getName().equals(ms[ms.length - 1])) {
                findOne = true;
                break;
            }
        }
        if (!findOne)
            throw new Exception("please provide valid metrics name in the format of group[.subgroup].metric_name. " + metricsName + " is not valid.");
    }
}

Example 5 with UserDefinedMetrics

use of com.yahoo.dba.perf.myperf.common.UserDefinedMetrics in project mysql_perf_analyzer by yahoo.

the class UdmController method processNewUDM.

private ModelAndView processNewUDM(HttpServletRequest req, HttpServletResponse resp) {
    boolean valid = true;
    String message = "OK";
    String cmd = req.getParameter("cmd");
    String name = req.getParameter("name");
    if (name != null)
        name = name.trim();
    String auto = req.getParameter("auto");
    String storage = req.getParameter("storage");
    String source = req.getParameter("source");
    String type = req.getParameter("type");
    String namecol = req.getParameter("namecol");
    String valcol = req.getParameter("valcol");
    String keycol = req.getParameter("keycol");
    String sql = req.getParameter("sql");
    String testdb = req.getParameter("testdb");
    String num = req.getParameter("num");
    UserDefinedMetrics udm = new UserDefinedMetrics(name);
    udm.setAuto("y".equals(auto));
    udm.setStoreInCommonTable("SHARED".equalsIgnoreCase(storage));
    udm.setSource(source);
    udm.setUdmType(type);
    udm.setNameCol(namecol);
    udm.setValueCol(valcol);
    udm.setKeyCol(keycol);
    udm.setSql(sql);
    int cnt = 0;
    try {
        cnt = Integer.parseInt(num);
    } catch (Exception ex) {
    }
    for (int i = 0; i < cnt; i++) {
        String mname = req.getParameter("mname_" + i);
        String mcol = req.getParameter("mcol_" + i);
        String minc = req.getParameter("minc_" + i);
        String mdata = req.getParameter("mdata_" + i);
        udm.addmetric(mname, mcol, "y".equalsIgnoreCase(minc), Metric.strToMetricDataType(mdata));
    }
    //now validation
    try {
        udm.validate();
    } catch (Exception ex) {
        valid = false;
        message = "Validation error: " + ex.getMessage();
    }
    if (//now run test and store
    valid) {
        try {
            this.testUDM(req, udm, testdb);
            //if reach here, add and store it.
            if ("publish".equalsIgnoreCase(cmd)) {
                this.frameworkContext.getMetricsDef().getUdmManager().addUDM(udm);
                this.frameworkContext.getMetricDb().addNewUDM(udm);
                this.frameworkContext.refreshMetricsList();
            }
        } catch (Exception ex) {
            logger.log(Level.WARNING, "Failed to test UDM", ex);
            valid = false;
            message = "Failed on test: " + ex.getMessage();
        }
    }
    ModelAndView mv = new ModelAndView(this.jsonView);
    if (req.getParameter("callback") != null && req.getParameter("callback").trim().length() > 0)
        //YUI datasource binding
        mv.addObject("callback", req.getParameter("callback"));
    if (//we response with udm name
    valid)
        mv.addObject("json_result", "{\"resp\":{\"status\":0, \"udm\": \"" + udm.getName() + "\", \"message\":\"" + message + "\"}}");
    else
        mv = this.respondFailure(message, req);
    return mv;
}