Examples with MetricsGroup com.yahoo.dba.perf.myperf.common.MetricsGroup used on opensource projects

Example 1 with MetricsGroup

use of com.yahoo.dba.perf.myperf.common.MetricsGroup in project mysql_perf_analyzer by yahoo.

the class MetricsDbBase method createTables.

private void createTables() {
    Connection conn = null;
    Statement stmt = null;
    try {
        conn = this.createConnection(false);
        for (Map.Entry<String, MetricsGroup> e : this.metricsGroups.entrySet()) {
            if (e.getValue().isStoreInCommonTable())
                //skip it. It will use generic table
                continue;
            createMetricsTable(e.getValue(), conn);
        }
        if (!DBUtils.hasTable(conn, schemaName, "METRIC_CODE")) {
            stmt = conn.createStatement();
            String ddl = this.buildMetricCodeDDL();
            logger.info("Create metric table METRIC_CODE: " + ddl);
            stmt.execute(ddl);
            stmt.execute(this.buildMetricCodeIndexDDL());
            DBUtils.close(stmt);
            logger.info("Created metric table METRIC_CODE");
        }
        if (!DBUtils.hasTable(conn, schemaName, "METRIC_GENERIC")) {
            stmt = conn.createStatement();
            String[] ddls = this.buildGenericMetricDDL();
            for (String ddl : ddls) {
                logger.info("Create metric table METRIC_GENERIC: " + ddl);
                stmt.execute(ddl);
            }
            DBUtils.close(stmt);
            logger.info("Created metric table METRIC_GENERIC");
        }
        if (!DBUtils.hasTable(conn, schemaName, "SNAPSHOTS")) {
            stmt = conn.createStatement();
            String[] ddls = this.buildSnapshotDDL();
            for (String ddl : ddls) {
                logger.info("Create metric table SNAPSHOTS: " + ddl);
                stmt.execute(ddl);
            }
            DBUtils.close(stmt);
            logger.info("Created metric table METRIC_GENERIC");
        }
        if (!DBUtils.hasTable(conn, schemaName, "ALERT")) {
            stmt = conn.createStatement();
            String ddl = this.buildAlertDDL();
            logger.info("Create alert table ALERT: " + ddl);
            stmt.execute(ddl);
            stmt.execute(this.buildAlertIndexDDL());
            DBUtils.close(stmt);
            logger.info("Created alert table ALERT");
        }
        if (!DBUtils.hasTable(conn, schemaName, "ALERT_SUBSCRIPT")) {
            stmt = conn.createStatement();
            String[] ddls = this.buildAlertSubScriptionDDL();
            for (String ddl : ddls) {
                logger.info("Create metric table ALERT_SUBSCRIPT: " + ddl);
                stmt.execute(ddl);
            }
            DBUtils.close(stmt);
            logger.info("Created metric table ALERT_SUBSCRIPT");
        }
        if (!DBUtils.hasTable(conn, schemaName, "METRICS_SUBSCRIPT")) {
            stmt = conn.createStatement();
            String[] ddls = this.buildMetricsSubscrptionDDL();
            for (String ddl : ddls) {
                logger.info("Create metric table METRICS_SUBSCRIPT: " + ddl);
                stmt.execute(ddl);
            }
            DBUtils.close(stmt);
            logger.info("Created metric table METRICS_SUBSCRIPT");
        }
        //buildAlertNotificationDDL
        if (!DBUtils.hasTable(conn, schemaName, "ALERT_NOTIFICATION")) {
            stmt = conn.createStatement();
            String[] ddls = this.buildAlertNotificationDDL();
            for (String ddl : ddls) {
                logger.info("Create metric table ALERT_NOTIFICATION: " + ddl);
                stmt.execute(ddl);
            }
            DBUtils.close(stmt);
            logger.info("Created metric table ALERT_NOTIFICATION");
        }
        if (!DBUtils.hasTable(conn, schemaName, DBINFO_TABLENAME)) {
            stmt = conn.createStatement();
            for (String ddl : this.buildHostDDL()) {
                if (ddl != null && !ddl.isEmpty()) {
                    logger.info("Create dbhost table:  " + ddl);
                    stmt.execute(ddl);
                    logger.info("Created dbhost table " + DBINFO_TABLENAME);
                }
            }
            DBUtils.close(stmt);
        }
        if (!DBUtils.hasTable(conn, schemaName, ALERTSETTING_TABLENAME)) {
            stmt = conn.createStatement();
            for (String ddl : this.buildAlertSettingDDL()) {
                if (ddl != null && !ddl.isEmpty()) {
                    logger.info("Create table:  " + ddl);
                    stmt.execute(ddl);
                    logger.info("Created table " + ALERTSETTING_TABLENAME);
                }
            }
            DBUtils.close(stmt);
        }
    } catch (Exception ex) {
        logger.log(Level.SEVERE, "Failed to check or create metric db", ex);
    } finally {
        DBUtils.close(conn);
    }
}

Example 2 with MetricsGroup

use of com.yahoo.dba.perf.myperf.common.MetricsGroup in project mysql_perf_analyzer by yahoo.

the class MetricsDbBase method setMetricsGroups.

/**
	   * This is used during startup
	   * @param mgs
	   */
public void setMetricsGroups(MetricsDefManager metricsDef) {
    synchronized (metricsDefLock) {
        String[] groupNames = metricsDef.getGroupNames();
        for (String grpName : groupNames) {
            MetricsGroup group = metricsDef.getGroupByName(grpName);
            //not supposed to be so
            if (group == null)
                continue;
            List<MetricsGroup> mgs = new ArrayList<MetricsGroup>();
            if (//no sub group, add self
            group.getSubGroups().size() == 0) {
                mgs.add(group);
            } else {
                for (MetricsGroup g : group.getSubGroups()) mgs.add(g);
            }
            for (MetricsGroup g : mgs) {
                String sinkName = g.getSinkTableName();
                this.metricsGroups.put(sinkName, g);
                //2014-02-14, change size to 10K
                this.dataQueues.put(sinkName, new java.util.concurrent.ArrayBlockingQueue<MetricsData>(10000));
                if (!g.isStoreInCommonTable())
                    this.insertSQL.put(sinkName, this.insertSQL(g));
                else {
                    String targetTable = g.getTargetTable();
                    if (targetTable == null || targetTable.isEmpty())
                        targetTable = "METRIC_GENERIC";
                    this.insertSQL.put((g.getDbType() + "_" + g.getGroupName()), "INSERT INTO " + targetTable + " (DBID, METRIC_ID, SNAP_ID, TS, VALUE) VALUES(?,?,?,?,?)");
                }
            }
        }
        for (Map.Entry<String, UserDefinedMetrics> entry : metricsDef.getUdmManager().getUdms().entrySet()) {
            MetricsGroup group = entry.getValue().getMetricsGroup();
            //not supposed to be so
            if (group == null)
                continue;
            String sinkName = group.getSinkTableName();
            this.metricsGroups.put(sinkName, group);
            //2014-02-14, change size to 10K	      
            this.dataQueues.put(sinkName, new java.util.concurrent.ArrayBlockingQueue<MetricsData>(10000));
            this.insertSQL.put(sinkName, this.insertSQL(group));
        }
    }
}

Example 3 with MetricsGroup

use of com.yahoo.dba.perf.myperf.common.MetricsGroup in project mysql_perf_analyzer by yahoo.

the class MetricsDbBase method store.

protected void store() {
    try {
        for (String s : this.dataQueues.keySet()) {
            java.util.concurrent.ArrayBlockingQueue<MetricsData> q = this.dataQueues.get(s);
            List<MetricsData> q2 = new ArrayList<MetricsData>(50);
            while (true) {
                MetricsData bufData = q.poll();
                if (//if nothing there, go to next queue.
                bufData == null)
                    break;
                //accumulate all data from the same queue
                q2.add(bufData);
            }
            if (q2.size() > 0) {
                java.sql.PreparedStatement stmt = null;
                logger.fine("Store " + q2.size() + " " + s + " metric records.");
                long currTime = System.currentTimeMillis();
                if (currTime - this.lastConnTime > 30000) {
                    DBUtils.close(storeConnection);
                    storeConnection = null;
                }
                if (storeConnection == null) {
                    storeConnection = this.createConnection(true);
                    this.lastConnTime = currTime;
                }
                //generic metrics
                MetricsGroup mg = this.metricsGroups.get(s);
                if (mg.isStoreInCommonTable()) {
                    storeGenericMetric(s, q2, storeConnection);
                    this.lastConnTime = System.currentTimeMillis();
                    logger.fine("Stored " + q2.size() + " " + s + " metric records.");
                    //go to next type
                    continue;
                }
                //builtin metrics
                try {
                    String sql = this.insertSQL.get(s);
                    stmt = storeConnection.prepareStatement(sql);
                    for (MetricsData mdata : q2) {
                        ByteBuffer buf = mdata.data;
                        int idx = 1;
                        int pos = 0;
                        //dbid
                        stmt.setInt(idx++, buf.getInt(pos));
                        //dbid
                        pos += 4;
                        //snap_id
                        stmt.setInt(idx++, buf.getInt(pos));
                        //snap_id
                        pos += 4;
                        if (mg.getKeyColumn() != null && !mg.getKeyColumn().isEmpty()) {
                            if (mdata.dataKey == null)
                                stmt.setNull(idx++, java.sql.Types.VARCHAR);
                            else
                                stmt.setString(idx++, mdata.dataKey);
                        }
                        //timestamp
                        stmt.setString(idx++, sdf.format(new java.util.Date(buf.getLong(pos))));
                        //timestamp
                        pos += 8;
                        //sql time
                        stmt.setInt(idx++, buf.getInt(pos));
                        //sql time
                        pos += 4;
                        List<Metric> ms = mg.getMetrics();
                        int len = ms.size();
                        for (int i = 0; i < len; i++) {
                            Metric m = ms.get(i);
                            if (m.getDataType() == MetricDataType.BYTE) {
                                stmt.setInt(idx++, buf.get(pos));
                                pos++;
                            } else if (m.getDataType() == MetricDataType.SHORT) {
                                stmt.setInt(idx++, buf.getShort(pos));
                                pos += 2;
                            } else if (m.getDataType() == MetricDataType.INT) {
                                stmt.setInt(idx++, buf.getInt(pos));
                                pos += 4;
                            } else if (m.getDataType() == MetricDataType.LONG) {
                                stmt.setLong(idx++, buf.getLong(pos));
                                pos += 8;
                            } else if (m.getDataType() == MetricDataType.FLOAT) {
                                stmt.setString(idx++, df.format(buf.getFloat(pos)));
                                pos += 4;
                            } else if (m.getDataType() == MetricDataType.DOUBLE) {
                                stmt.setString(idx++, df.format(buf.getDouble(pos)));
                                pos += 8;
                            }
                        }
                        //one row
                        stmt.execute();
                        stmt.clearBatch();
                    }
                //for(MetricsData mdata:q2)
                } catch (Exception ex) {
                    logger.log(Level.WARNING, "Exception when store metrics: " + s, ex);
                    if (ex instanceof com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException) {
                    //constraint error, skip
                    }
                }
                //try{
                //  stmt.executeBatch();
                //}catch(Exception iex)
                //{
                // logger.warning("Failed: "+sql);
                //  throw iex;
                //}
                //storeConnection.commit();
                this.lastConnTime = System.currentTimeMillis();
                stmt.close();
                stmt = null;
                logger.fine("Stored " + q2.size() + " " + s + " metric records.");
            }
        }
    } catch (Exception ex) {
        logger.log(Level.WARNING, "Exception when store metrics", ex);
        if (ex instanceof com.mysql.jdbc.exceptions.jdbc4.MySQLNonTransientConnectionException) {
            DBUtils.close(storeConnection);
            storeConnection = null;
        }
        if (storeConnection != null) {
            try {
                storeConnection.rollback();
            } catch (Exception iex) {
            }
        }
    } finally {
    //DBUtils.close(stmt);
    //DBUtils.close(conn);
    }
}

Example 4 with MetricsGroup

use of com.yahoo.dba.perf.myperf.common.MetricsGroup in project mysql_perf_analyzer by yahoo.

the class MetricsRetentionTask method run.

@Override
public void run() {
    Thread.currentThread().setName("MetricsRetentionTask");
    logger.info("Starting metrics purge job");
    if (this.context.getMetricDb() == null) {
        logger.info("MetricsDB has yet to set.");
        return;
    }
    //get all dbids
    List<Integer> ids = new ArrayList<Integer>();
    if (this.dbidToPurge == null) {
        for (Map.Entry<String, DBGroupInfo> e : context.getDbInfoManager().getClusters().entrySet()) {
            DBGroupInfo g = e.getValue();
            for (DBInstanceInfo i : g.getInstances()) {
                ids.add(i.getDbid());
            }
        }
    } else {
        for (int id : this.dbidToPurge) ids.add(id);
    }
    Calendar c = Calendar.getInstance();
    c.add(Calendar.DATE, -this.renentionDays);
    Date dt = c.getTime();
    //now get current timestamp
    java.text.SimpleDateFormat sdf = new java.text.SimpleDateFormat("yyyyMMddHHmmss");
    sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
    long endDate = Long.parseLong(sdf.format(dt));
    List<MetricsGroup> mgs = new ArrayList<MetricsGroup>();
    String[] groupNames = this.context.getMetricsDef().getGroupNames();
    for (String grpName : groupNames) {
        MetricsGroup group = this.context.getMetricsDef().getGroupByName(grpName);
        //not supposed to be so
        if (group == null)
            continue;
        if (//no sub group, add self
        group.getSubGroups().size() == 0) {
            mgs.add(group);
        } else {
            for (MetricsGroup g : group.getSubGroups()) mgs.add(g);
        }
    }
    for (Map.Entry<String, UserDefinedMetrics> entry : this.context.getMetricsDef().getUdmManager().getUdms().entrySet()) {
        MetricsGroup group = entry.getValue().getMetricsGroup();
        //not supposed to be so
        if (group == null)
            continue;
        mgs.add(group);
    }
    for (int dbid : ids) {
        logger.info("Check and purge db: " + dbid);
        for (MetricsGroup g : mgs) {
            if (this.dbidToPurge != null)
                this.context.getMetricDb().purgeAll(g.getSinkTableName(), dbid);
            else
                this.context.getMetricDb().purge(g.getSinkTableName(), dbid, endDate);
        }
    }
    java.text.SimpleDateFormat sdf2 = new java.text.SimpleDateFormat("yyyyMMdd");
    sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
    purgeAlertReports(Integer.parseInt(sdf2.format(dt)));
    logger.info("Ended metrics purge job");
    if (this.dbidToPurge == null)
        this.context.getMetricDb().purgeAlerts(endDate);
    //for now, we only do it once a day.
    //TODO keep consistency with DB add/update/delete
    logger.info("Retention job done.");
}

Example 5 with MetricsGroup

use of com.yahoo.dba.perf.myperf.common.MetricsGroup in project mysql_perf_analyzer by yahoo.

the class MySQLMetricsDb method storeGenericMetric.

@Override
protected void storeGenericMetric(String s, List<MetricsData> q2, Connection conn) throws SQLException {
    Statement stmt = null;
    MetricsGroup mg = this.metricsGroups.get(s);
    StringBuilder sb = new StringBuilder();
    String targetTable = mg != null ? mg.getTargetTable() : null;
    if (targetTable == null || targetTable.isEmpty())
        targetTable = "METRIC_GENERIC";
    sb.append("INSERT INTO " + targetTable + " (DBID, METRIC_ID, SNAP_ID, TS, VALUE) VALUES");
    try {
        stmt = conn.createStatement();
        int cnt = 0;
        //hard coded for now
        int batchSize = 100;
        for (MetricsData dbuf : q2) {
            ByteBuffer buf = dbuf.data;
            int idx = 1;
            int pos = 0;
            if (cnt > 0)
                sb.append(",");
            sb.append("(");
            sb.append(buf.getInt(pos));
            pos += 4;
            sb.append(",").append(buf.getInt(pos));
            idx++;
            pos += 4;
            sb.append(",").append(buf.getInt(pos));
            idx++;
            pos += 4;
            //TODO sdf thread safety
            sb.append(",").append(sdf.format(new java.util.Date(buf.getLong(pos))));
            //TODO sdf thread safety
            idx++;
            //TODO sdf thread safety
            pos += 8;
            sb.append(",").append(df.format(buf.getDouble(pos))).append(")");
            idx++;
            cnt++;
            if (cnt >= batchSize) {
                //logger.info("Batch statement: "+sb.toString());
                stmt.execute(sb.toString());
                cnt = 0;
                //conn.commit();
                sb.setLength(0);
                sb.append("INSERT INTO " + targetTable + " (DBID, METRIC_ID, TS, VALUE) VALUES");
            }
        }
        //for loop
        if (cnt > 0) {
            //logger.info("Batch statement: "+sb.toString());
            stmt.execute(sb.toString());
            //conn.commit();
            sb.setLength(0);
        }
        stmt.close();
        stmt = null;
    } finally {
        DBUtils.close(stmt);
    }
}