Examples with Metric com.yahoo.dba.perf.myperf.common.Metric used on opensource projects

Example 1 with Metric

use of com.yahoo.dba.perf.myperf.common.Metric in project mysql_perf_analyzer by yahoo.

the class MetricsDbBase method retrieveMetrics.

public ResultList retrieveMetrics(String metricGroupName, Metric[] metrics, boolean hasKeyColumn, int dbid, long startDate, long endDate, boolean agg) {
    String[] ms = new String[metrics.length];
    for (int i = 0; i < metrics.length; i++) ms[i] = metrics[i].getName();
    if (!agg)
        return retrieveMetrics(metricGroupName, ms, hasKeyColumn, dbid, startDate, endDate);
    int[] snaps = this.getSnapshostRange(startDate, endDate);
    //no data
    if (snaps == null)
        return null;
    if (//not specify the metrics? Get all
    metrics == null || metrics.length == 0)
        return retrieveMetrics(metricGroupName, dbid, startDate, endDate);
    //later, connection pooling
    ResultList rList = null;
    Connection conn = null;
    PreparedStatement stmt = null;
    ResultSet rs = null;
    //build select list
    StringBuilder sb = new StringBuilder();
    //build select list
    StringBuilder grpBy = new StringBuilder();
    sb.append("SNAP_ID");
    sb.append(", TS");
    for (Metric me : metrics) {
        if (me.isIncremental())
            sb.append(", sum(");
        else
            sb.append(", avg(");
        sb.append(me.getName()).append(") ").append(me.getName());
    }
    String sql = "select " + sb.toString() + " from " + metricGroupName + " where dbid=? and snap_id between ? and ? group by snap_id, ts order by snap_id";
    //String sql = "select * from "+metricGroupName+" where dbid=?";
    logger.log(Level.INFO, "To retrieve metrics " + metricGroupName + ", metrics (" + sb.toString() + ") for db " + dbid + " with time range (" + startDate + ", " + endDate + "), snap (" + snaps[0] + ", " + snaps[1] + ")");
    try {
        conn = createConnection(true);
        stmt = conn.prepareStatement(sql);
        stmt.setFetchSize(1000);
        //stmt.setMaxRows(5000);
        stmt.setInt(1, dbid);
        stmt.setInt(2, snaps[0]);
        stmt.setInt(3, snaps[1]);
        rs = stmt.executeQuery();
        rList = ResultListUtil.fromSqlResultSet(rs, 5000);
    } catch (Exception ex) {
        logger.log(Level.SEVERE, "Failed to retrieve metrics " + metricGroupName + " for db " + dbid + " with time range (" + startDate + ", " + endDate + ")", ex);
    } finally {
        DBUtils.close(stmt);
        DBUtils.close(conn);
    }
    return rList;
}

Example 2 with Metric

use of com.yahoo.dba.perf.myperf.common.Metric in project mysql_perf_analyzer by yahoo.

the class MetricsDbBase method insertSQL.

/**
	   * Build insert statement.
	   * TODO db specific optimization
	   * @param mg
	   * @return
	   */
private String insertSQL(MetricsGroup mg) {
    StringBuilder sb = new StringBuilder();
    sb.append("insert ignore into ");
    if (mg.isUdmFlagged())
        sb.append("UDM");
    else
        sb.append(mg.getDbType().toUpperCase());
    sb.append("_").append(mg.getGroupName().toUpperCase()).append(" (").append("DBID, SNAP_ID");
    if (mg.getKeyColumn() != null && !mg.getKeyColumn().isEmpty()) {
        //add additional column to store key
        sb.append(", KEY_COLUMN");
    }
    sb.append(", TS, SQL_TIME");
    for (Metric m : mg.getMetrics()) {
        sb.append(",");
        sb.append(m.getName().toUpperCase());
    }
    sb.append(") values (?,?,?,?");
    if (mg.getKeyColumn() != null && !mg.getKeyColumn().isEmpty()) {
        //add additional column to store key
        sb.append(",?");
    }
    for (Metric m : mg.getMetrics()) {
        sb.append(",?");
    }
    sb.append(")");
    return sb.toString();
}

Example 3 with Metric

use of com.yahoo.dba.perf.myperf.common.Metric in project mysql_perf_analyzer by yahoo.

the class DerbyMetricsDb method buildDDL.

@Override
protected String buildDDL(MetricsGroup mg) {
    StringBuilder sb = new StringBuilder();
    String tblName = mg.getSinkTableName();
    sb.append("CREATE TABLE ");
    sb.append(tblName);
    sb.append(" (").append("DBID INT, SNAP_ID INT");
    if (mg.getKeyColumn() != null && !mg.getKeyColumn().isEmpty()) {
        //add additional column to store key
        sb.append(", KEY_COLUMN VARCHAR(255)");
    }
    sb.append(", TS BIGINT, SQL_TIME INT");
    for (Metric m : mg.getMetrics()) {
        sb.append(",");
        sb.append(m.getName().toUpperCase()).append(" ");
        if (m.getDataType() == MetricDataType.BYTE)
            sb.append("SMALLINT");
        else if (m.getDataType() == MetricDataType.SHORT)
            sb.append("SMALLINT");
        else if (m.getDataType() == MetricDataType.INT)
            sb.append("INT");
        else if (m.getDataType() == MetricDataType.LONG)
            sb.append("BIGINT");
        else if (m.getDataType() == MetricDataType.FLOAT)
            sb.append("DECIMAL(22,7)");
        else if (m.getDataType() == MetricDataType.DOUBLE)
            sb.append("DECIMAL(22,7)");
    }
    if (mg.getKeyColumn() != null && !mg.getKeyColumn().isEmpty())
        sb.append(", PRIMARY KEY(DBID, SNAP_ID, KEY_COLUMN))");
    else
        sb.append(", PRIMARY KEY(DBID, SNAP_ID))");
    return sb.toString();
}

Example 4 with Metric

use of com.yahoo.dba.perf.myperf.common.Metric in project mysql_perf_analyzer by yahoo.

the class MetricsDbBase method store.

protected void store() {
    try {
        for (String s : this.dataQueues.keySet()) {
            java.util.concurrent.ArrayBlockingQueue<MetricsData> q = this.dataQueues.get(s);
            List<MetricsData> q2 = new ArrayList<MetricsData>(50);
            while (true) {
                MetricsData bufData = q.poll();
                if (//if nothing there, go to next queue.
                bufData == null)
                    break;
                //accumulate all data from the same queue
                q2.add(bufData);
            }
            if (q2.size() > 0) {
                java.sql.PreparedStatement stmt = null;
                logger.fine("Store " + q2.size() + " " + s + " metric records.");
                long currTime = System.currentTimeMillis();
                if (currTime - this.lastConnTime > 30000) {
                    DBUtils.close(storeConnection);
                    storeConnection = null;
                }
                if (storeConnection == null) {
                    storeConnection = this.createConnection(true);
                    this.lastConnTime = currTime;
                }
                //generic metrics
                MetricsGroup mg = this.metricsGroups.get(s);
                if (mg.isStoreInCommonTable()) {
                    storeGenericMetric(s, q2, storeConnection);
                    this.lastConnTime = System.currentTimeMillis();
                    logger.fine("Stored " + q2.size() + " " + s + " metric records.");
                    //go to next type
                    continue;
                }
                //builtin metrics
                try {
                    String sql = this.insertSQL.get(s);
                    stmt = storeConnection.prepareStatement(sql);
                    for (MetricsData mdata : q2) {
                        ByteBuffer buf = mdata.data;
                        int idx = 1;
                        int pos = 0;
                        //dbid
                        stmt.setInt(idx++, buf.getInt(pos));
                        //dbid
                        pos += 4;
                        //snap_id
                        stmt.setInt(idx++, buf.getInt(pos));
                        //snap_id
                        pos += 4;
                        if (mg.getKeyColumn() != null && !mg.getKeyColumn().isEmpty()) {
                            if (mdata.dataKey == null)
                                stmt.setNull(idx++, java.sql.Types.VARCHAR);
                            else
                                stmt.setString(idx++, mdata.dataKey);
                        }
                        //timestamp
                        stmt.setString(idx++, sdf.format(new java.util.Date(buf.getLong(pos))));
                        //timestamp
                        pos += 8;
                        //sql time
                        stmt.setInt(idx++, buf.getInt(pos));
                        //sql time
                        pos += 4;
                        List<Metric> ms = mg.getMetrics();
                        int len = ms.size();
                        for (int i = 0; i < len; i++) {
                            Metric m = ms.get(i);
                            if (m.getDataType() == MetricDataType.BYTE) {
                                stmt.setInt(idx++, buf.get(pos));
                                pos++;
                            } else if (m.getDataType() == MetricDataType.SHORT) {
                                stmt.setInt(idx++, buf.getShort(pos));
                                pos += 2;
                            } else if (m.getDataType() == MetricDataType.INT) {
                                stmt.setInt(idx++, buf.getInt(pos));
                                pos += 4;
                            } else if (m.getDataType() == MetricDataType.LONG) {
                                stmt.setLong(idx++, buf.getLong(pos));
                                pos += 8;
                            } else if (m.getDataType() == MetricDataType.FLOAT) {
                                stmt.setString(idx++, df.format(buf.getFloat(pos)));
                                pos += 4;
                            } else if (m.getDataType() == MetricDataType.DOUBLE) {
                                stmt.setString(idx++, df.format(buf.getDouble(pos)));
                                pos += 8;
                            }
                        }
                        //one row
                        stmt.execute();
                        stmt.clearBatch();
                    }
                //for(MetricsData mdata:q2)
                } catch (Exception ex) {
                    logger.log(Level.WARNING, "Exception when store metrics: " + s, ex);
                    if (ex instanceof com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException) {
                    //constraint error, skip
                    }
                }
                //try{
                //  stmt.executeBatch();
                //}catch(Exception iex)
                //{
                // logger.warning("Failed: "+sql);
                //  throw iex;
                //}
                //storeConnection.commit();
                this.lastConnTime = System.currentTimeMillis();
                stmt.close();
                stmt = null;
                logger.fine("Stored " + q2.size() + " " + s + " metric records.");
            }
        }
    } catch (Exception ex) {
        logger.log(Level.WARNING, "Exception when store metrics", ex);
        if (ex instanceof com.mysql.jdbc.exceptions.jdbc4.MySQLNonTransientConnectionException) {
            DBUtils.close(storeConnection);
            storeConnection = null;
        }
        if (storeConnection != null) {
            try {
                storeConnection.rollback();
            } catch (Exception iex) {
            }
        }
    } finally {
    //DBUtils.close(stmt);
    //DBUtils.close(conn);
    }
}

Example 5 with Metric

use of com.yahoo.dba.perf.myperf.common.Metric in project mysql_perf_analyzer by yahoo.

the class MetricsDbBase method buildDDL.

protected String buildDDL(MetricsGroup mg) {
    StringBuilder sb = new StringBuilder();
    String tblName = mg.getSinkTableName();
    sb.append("CREATE TABLE ");
    sb.append(tblName);
    sb.append(" (").append("DBID INT, SNAP_ID INT");
    if (mg.getKeyColumn() != null && !mg.getKeyColumn().isEmpty()) {
        //add additional column to store key
        sb.append(", KEY_COLUMN VARCHAR(255)");
    }
    sb.append(", TS BIGINT, SQL_TIME INT");
    for (Metric m : mg.getMetrics()) {
        sb.append(",");
        sb.append(m.getName().toUpperCase()).append(" ");
        if (m.getDataType() == MetricDataType.BYTE)
            sb.append("TINYINT");
        else if (m.getDataType() == MetricDataType.SHORT)
            sb.append("SMALLINT");
        else if (m.getDataType() == MetricDataType.INT)
            sb.append("INT");
        else if (m.getDataType() == MetricDataType.LONG)
            sb.append("BIGINT");
        else if (m.getDataType() == MetricDataType.FLOAT)
            sb.append("DECIMAL(22,7)");
        else if (m.getDataType() == MetricDataType.DOUBLE)
            sb.append("DECIMAL(22,7)");
    }
    if (mg.getKeyColumn() != null && !mg.getKeyColumn().isEmpty())
        sb.append(", PRIMARY KEY(DBID, SNAP_ID, KEY_COLUMN))");
    else
        sb.append(", PRIMARY KEY(DBID, SNAP_ID))");
    return sb.toString();
}