Examples with PageMetric datawave.microservice.querymetric.BaseQueryMetric.PageMetric used on opensource projects

Example 6 with PageMetric

use of datawave.microservice.querymetric.BaseQueryMetric.PageMetric in project datawave-query-metric-service by NationalSecurityAgency.

the class QueryMetricTest method testPageMetricParsingLegacy1.

@Test
public void testPageMetricParsingLegacy1() {
    PageMetric pmRef1 = new PageMetric(null, null, 2500, 2000, 3500, 3600, -1, 2200, 3000, 10000);
    // pageSize/returnTime/callTime/serializationTime/bytesWritten/pageRequested/pageReturned
    String pmText1 = "2500/2000/2200/3000/10000/3500/3600";
    PageMetric pm1 = PageMetric.parse(pmText1);
    assertEquals("page metrics not equal", pmRef1, pm1);
}

Example 7 with PageMetric

use of datawave.microservice.querymetric.BaseQueryMetric.PageMetric in project datawave-query-metric-service by NationalSecurityAgency.

the class QueryMetricsDetailListResponse method getMainContent.

@Override
public String getMainContent() {
    StringBuilder builder = new StringBuilder(), pageTimesBuilder = new StringBuilder();
    builder.append("<table>\n");
    builder.append("<tr>");
    builder.append("<th>Visibility</th><th>Query Date</th><th>User</th><th>UserDN</th><th>Proxy Server(s)</th><th>Query ID</th><th>Query Type</th>");
    builder.append("<th>Query Logic</th><th id=\"query-header\">Query</th><th>Query Plan</th><th>Query Name</th><th>Begin Date</th><th>End Date</th><th>Parameters</th><th>Query Auths</th>");
    builder.append("<th>Server</th>");
    builder.append("<th>Predictions</th>");
    builder.append("<th>Login Time (ms)</th>");
    builder.append("<th>Query Setup Time (ms)</th><th>Query Setup Call Time (ms)</th><th>Number Pages</th><th>Number Results</th>");
    builder.append("<th>Doc Ranges</th><th>FI Ranges</th>");
    builder.append("<th>Sources</th><th>Next Calls</th><th>Seek Calls</th><th>Yield Count</th><th>Version</th>");
    builder.append("<th>Total Page Time (ms)</th><th>Total Page Call Time (ms)</th><th>Total Page Serialization Time (ms)</th>");
    builder.append("<th>Total Page Bytes Sent (uncompressed)</th><th>Lifecycle</th><th>Elapsed Time</th><th>Error Code</th><th>Error Message</th>");
    builder.append("</tr>\n");
    pageTimesBuilder.append("<table>\n");
    pageTimesBuilder.append("<tr><th>Page number</th><th>Page requested</th><th>Page returned</th><th>Response time (ms)</th>");
    pageTimesBuilder.append("<th>Page size</th><th>Call time (ms)</th><th>Login time (ms)</th><th>Serialization time (ms)</th>");
    pageTimesBuilder.append("<th>Bytes written (uncompressed)</th></tr>");
    TreeMap<Date, QueryMetric> metricMap = new TreeMap<Date, QueryMetric>(Collections.reverseOrder());
    for (QueryMetric metric : this.getResult()) {
        metricMap.put(metric.getCreateDate(), metric);
    }
    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd HHmmss");
    int x = 0;
    for (QueryMetric metric : metricMap.values()) {
        // highlight alternating rows
        if (x % 2 == 0) {
            builder.append("<tr class=\"highlight\">\n");
        } else {
            builder.append("<tr>\n");
        }
        x++;
        builder.append("<td>").append(metric.getColumnVisibility()).append("</td>");
        builder.append("<td style=\"min-width:125px !important;\">").append(sdf.format(metric.getCreateDate())).append("</td>");
        builder.append("<td>").append(metric.getUser()).append("</td>");
        String userDN = metric.getUserDN();
        builder.append("<td style=\"min-width:500px !important;\">").append(userDN == null ? "" : userDN).append("</td>");
        String proxyServers = metric.getProxyServers() == null ? "" : StringUtils.join(metric.getProxyServers(), "<BR/>");
        builder.append("<td>").append(proxyServers).append("</td>");
        builder.append("<td>").append(metric.getQueryId()).append("</td>");
        builder.append("<td>").append(metric.getQueryType()).append("</td>");
        builder.append("<td>").append(metric.getQueryLogic()).append("</td>");
        builder.append("<td style=\"word-wrap: break-word;\">").append(StringEscapeUtils.escapeHtml4(metric.getQuery())).append("</td>");
        builder.append("<td style=\"word-wrap: break-word;\">").append(StringEscapeUtils.escapeHtml4(metric.getPlan())).append("</td>");
        builder.append("<td>").append(metric.getQueryName()).append("</td>");
        String beginDate = metric.getBeginDate() == null ? "" : sdf.format(metric.getBeginDate());
        builder.append("<td style=\"min-width:125px !important;\">").append(beginDate).append("</td>");
        String endDate = metric.getEndDate() == null ? "" : sdf.format(metric.getEndDate());
        builder.append("<td style=\"min-width:125px !important;\">").append(endDate).append("</td>");
        Set<Parameter> parameters = metric.getParameters();
        builder.append("<td>").append(parameters == null ? "" : toParametersString(parameters)).append("</td>");
        String queryAuths = metric.getQueryAuthorizations() == null ? "" : metric.getQueryAuthorizations().replaceAll(",", " ");
        builder.append("<td style=\"word-wrap: break-word; min-width:300px !important;\">").append(queryAuths).append("</td>");
        builder.append("<td>").append(metric.getHost()).append("</td>");
        if (metric.getPredictions() != null && !metric.getPredictions().isEmpty()) {
            builder.append("<td>");
            String delimiter = "";
            List<Prediction> predictions = new ArrayList<Prediction>(metric.getPredictions());
            Collections.sort(predictions);
            for (Prediction prediction : predictions) {
                builder.append(delimiter).append(prediction.getName()).append(" = ").append(prediction.getPrediction());
                delimiter = "<br>";
            }
        } else {
            builder.append("<td/>");
        }
        builder.append("<td>").append(numToString(metric.getLoginTime(), 0)).append("</td>");
        builder.append("<td>").append(metric.getSetupTime()).append("</td>");
        builder.append("<td>").append(numToString(metric.getCreateCallTime(), 0)).append("</td>\n");
        builder.append("<td>").append(metric.getNumPages()).append("</td>");
        builder.append("<td>").append(metric.getNumResults()).append("</td>");
        builder.append("<td>").append(metric.getDocRanges()).append("</td>");
        builder.append("<td>").append(metric.getFiRanges()).append("</td>");
        builder.append("<td>").append(metric.getSourceCount()).append("</td>");
        builder.append("<td>").append(metric.getNextCount()).append("</td>");
        builder.append("<td>").append(metric.getSeekCount()).append("</td>");
        builder.append("<td>").append(metric.getYieldCount()).append("</td>");
        builder.append("<td>").append(metric.getVersion()).append("</td>");
        long count = 0l;
        long callTime = 0l;
        long serializationTime = 0l;
        long bytesSent = 0l;
        int y = 0;
        for (PageMetric p : metric.getPageTimes()) {
            count += p.getReturnTime();
            callTime += (p.getCallTime()) == -1 ? 0 : p.getCallTime();
            serializationTime += (p.getSerializationTime()) == -1 ? 0 : p.getSerializationTime();
            bytesSent += (p.getBytesWritten()) == -1 ? 0 : p.getBytesWritten();
            if (y % 2 == 0) {
                pageTimesBuilder.append("<tr class=\"highlight\">");
            } else {
                pageTimesBuilder.append("<tr>");
            }
            y++;
            String pageRequestedStr = "";
            long pageRequested = p.getPageRequested();
            if (pageRequested > 0) {
                pageRequestedStr = sdf.format(new Date(pageRequested));
            }
            String pageReturnedStr = "";
            long pageReturned = p.getPageReturned();
            if (pageReturned > 0) {
                pageReturnedStr = sdf.format(new Date(pageReturned));
            }
            pageTimesBuilder.append("<td>").append(numToString(p.getPageNumber(), 1)).append("</td>");
            pageTimesBuilder.append("<td>").append(pageRequestedStr).append("</td>");
            pageTimesBuilder.append("<td>").append(pageReturnedStr).append("</td>");
            pageTimesBuilder.append("<td>").append(p.getReturnTime()).append("</td>");
            pageTimesBuilder.append("<td>").append(p.getPagesize()).append("</td>");
            pageTimesBuilder.append("<td>").append(numToString(p.getCallTime(), 0)).append("</td>");
            pageTimesBuilder.append("<td>").append(numToString(p.getLoginTime(), 0)).append("</td>");
            pageTimesBuilder.append("<td>").append(numToString(p.getSerializationTime(), 0)).append("</td>");
            pageTimesBuilder.append("<td>").append(numToString(p.getBytesWritten(), 0)).append("</td></tr>");
        }
        builder.append("<td>").append(count).append("</td>\n");
        builder.append("<td>").append(numToString(callTime, 0)).append("</td>\n");
        builder.append("<td>").append(numToString(serializationTime, 0)).append("</td>\n");
        builder.append("<td>").append(numToString(bytesSent, 0)).append("</td>\n");
        builder.append("<td>").append(metric.getLifecycle()).append("</td>");
        builder.append("<td>").append(metric.getElapsedTime()).append("</td>");
        String errorCode = metric.getErrorCode();
        builder.append("<td style=\"word-wrap: break-word;\">").append((errorCode == null) ? "" : StringEscapeUtils.escapeHtml4(errorCode)).append("</td>");
        String errorMessage = metric.getErrorMessage();
        builder.append("<td style=\"word-wrap: break-word;\">").append((errorMessage == null) ? "" : StringEscapeUtils.escapeHtml4(errorMessage)).append("</td>");
        builder.append("\n</tr>\n");
    }
    builder.append("</table>\n<br/>\n");
    pageTimesBuilder.append("</table>\n");
    builder.append(pageTimesBuilder);
    return builder.toString();
}

Example 8 with PageMetric

use of datawave.microservice.querymetric.BaseQueryMetric.PageMetric in project datawave-query-metric-service by NationalSecurityAgency.

the class QueryMetricCombiner method combineMetrics.

public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricType metricType) throws Exception {
    T combinedMetric = updatedQueryMetric;
    // new metrics coming in may be complete or partial updates
    if (cachedQueryMetric != null) {
        // duplicate cachedQueryMetric so that we leave that object unchanged and return a combined metric
        combinedMetric = (T) cachedQueryMetric.duplicate();
        // only update once
        if (combinedMetric.getQueryType() == null && updatedQueryMetric.getQueryType() != null) {
            combinedMetric.setQueryType(updatedQueryMetric.getQueryType());
        }
        // only update once
        if (combinedMetric.getUser() == null && updatedQueryMetric.getUser() != null) {
            combinedMetric.setUser(updatedQueryMetric.getUser());
        }
        // only update once
        if (combinedMetric.getUserDN() == null && updatedQueryMetric.getUserDN() != null) {
            combinedMetric.setUserDN(updatedQueryMetric.getUserDN());
        }
        // keep the earliest create date
        long cachedCreate = combinedMetric.getCreateDate() == null ? Long.MAX_VALUE : combinedMetric.getCreateDate().getTime();
        long updatedCreate = updatedQueryMetric.getCreateDate() == null ? Long.MAX_VALUE : updatedQueryMetric.getCreateDate().getTime();
        if (updatedCreate < cachedCreate) {
            combinedMetric.setCreateDate(updatedQueryMetric.getCreateDate());
        }
        // only update once
        if (combinedMetric.getQuery() == null && updatedQueryMetric.getQuery() != null) {
            combinedMetric.setQuery(updatedQueryMetric.getQuery());
        }
        // only update once
        if (combinedMetric.getHost() == null && updatedQueryMetric.getHost() != null) {
            combinedMetric.setHost(updatedQueryMetric.getHost());
        }
        // Map page numbers to page metrics and update
        Map<Long, PageMetric> storedPagesByPageNumMap = new TreeMap<>();
        Map<String, PageMetric> storedPagesByUuidMap = new TreeMap<>();
        if (combinedMetric.getPageTimes() != null) {
            combinedMetric.getPageTimes().forEach(pm -> {
                storedPagesByPageNumMap.put(pm.getPageNumber(), pm);
                if (pm.getPageUuid() != null) {
                    storedPagesByUuidMap.put(pm.getPageUuid(), pm);
                }
            });
        }
        // combine all of the page metrics from the cached metric and the updated metric
        if (updatedQueryMetric.getPageTimes() != null) {
            long pageNum = getLastPageNumber(combinedMetric) + 1;
            for (PageMetric updatedPage : updatedQueryMetric.getPageTimes()) {
                PageMetric storedPage = null;
                if (updatedPage.getPageUuid() != null) {
                    storedPage = storedPagesByUuidMap.get(updatedPage.getPageUuid());
                }
                if (metricType.equals(QueryMetricType.DISTRIBUTED)) {
                    if (storedPage != null) {
                        // updatedPage found by pageUuid
                        updatedPage = combinePageMetrics(updatedPage, storedPage);
                        storedPagesByPageNumMap.put(updatedPage.getPageNumber(), updatedPage);
                    } else {
                        // assume that this is the next page in sequence
                        updatedPage.setPageNumber(pageNum);
                        storedPagesByPageNumMap.put(pageNum, updatedPage);
                        pageNum++;
                    }
                } else {
                    if (storedPage == null) {
                        storedPage = storedPagesByPageNumMap.get(updatedPage.getPageNumber());
                    }
                    if (storedPage != null) {
                        updatedPage = combinePageMetrics(updatedPage, storedPage);
                    }
                    // page metrics are mapped to their page number to prevent duplicates
                    storedPagesByPageNumMap.put(updatedPage.getPageNumber(), updatedPage);
                }
            }
        }
        combinedMetric.setPageTimes(new ArrayList<>(storedPagesByPageNumMap.values()));
        // only update once
        if (combinedMetric.getProxyServers() == null && updatedQueryMetric.getProxyServers() != null) {
            combinedMetric.setProxyServers(updatedQueryMetric.getProxyServers());
        }
        // only update once
        if (combinedMetric.getErrorMessage() == null && updatedQueryMetric.getErrorMessage() != null) {
            combinedMetric.setErrorMessage(updatedQueryMetric.getErrorMessage());
        }
        // only update once
        if (combinedMetric.getErrorCode() == null && updatedQueryMetric.getErrorCode() != null) {
            combinedMetric.setErrorCode(updatedQueryMetric.getErrorCode());
        }
        // use updated lifecycle unless trying to update a final lifecycle with a non-final lifecycle
        if ((combinedMetric.isLifecycleFinal() && !updatedQueryMetric.isLifecycleFinal()) == false) {
            combinedMetric.setLifecycle(updatedQueryMetric.getLifecycle());
        }
        // only update once
        if (combinedMetric.getQueryAuthorizations() == null && updatedQueryMetric.getQueryAuthorizations() != null) {
            combinedMetric.setQueryAuthorizations(updatedQueryMetric.getQueryAuthorizations());
        }
        // only update once
        if (combinedMetric.getBeginDate() == null && updatedQueryMetric.getBeginDate() != null) {
            combinedMetric.setBeginDate(updatedQueryMetric.getBeginDate());
        }
        // only update once
        if (combinedMetric.getEndDate() == null && updatedQueryMetric.getEndDate() != null) {
            combinedMetric.setEndDate(updatedQueryMetric.getEndDate());
        }
        // only update once
        if (combinedMetric.getPositiveSelectors() == null && updatedQueryMetric.getPositiveSelectors() != null) {
            combinedMetric.setPositiveSelectors(updatedQueryMetric.getPositiveSelectors());
        }
        // only update once
        if (combinedMetric.getNegativeSelectors() == null && updatedQueryMetric.getNegativeSelectors() != null) {
            combinedMetric.setNegativeSelectors(updatedQueryMetric.getNegativeSelectors());
        }
        if (updatedQueryMetric.getLastUpdated() != null) {
            // keep the latest last updated date
            if (combinedMetric.getLastUpdated() == null || (updatedQueryMetric.getLastUpdated().getTime() > combinedMetric.getLastUpdated().getTime())) {
                combinedMetric.setLastUpdated(updatedQueryMetric.getLastUpdated());
            }
        }
        // only update once
        if (combinedMetric.getColumnVisibility() == null && updatedQueryMetric.getColumnVisibility() != null) {
            combinedMetric.setColumnVisibility(updatedQueryMetric.getColumnVisibility());
        }
        // only update once
        if (combinedMetric.getQueryLogic() == null && updatedQueryMetric.getQueryLogic() != null) {
            combinedMetric.setQueryLogic(updatedQueryMetric.getQueryLogic());
        }
        // only update once
        if (combinedMetric.getQueryName() == null && updatedQueryMetric.getQueryName() != null) {
            combinedMetric.setQueryName(updatedQueryMetric.getQueryName());
        }
        // only update once
        if (combinedMetric.getParameters() == null && updatedQueryMetric.getParameters() != null) {
            combinedMetric.setParameters(updatedQueryMetric.getParameters());
        }
        // only update once
        if (combinedMetric.getSetupTime() > -1) {
            combinedMetric.setSetupTime(updatedQueryMetric.getSetupTime());
        }
        // only update once
        if (combinedMetric.getCreateCallTime() > -1) {
            combinedMetric.setCreateCallTime(updatedQueryMetric.getCreateCallTime());
        }
        // only update once
        if (combinedMetric.getLoginTime() > -1) {
            combinedMetric.setLoginTime(updatedQueryMetric.getLoginTime());
        }
        if (metricType.equals(QueryMetricType.DISTRIBUTED)) {
            combinedMetric.setSourceCount(combinedMetric.getSourceCount() + updatedQueryMetric.getSourceCount());
            combinedMetric.setNextCount(combinedMetric.getNextCount() + updatedQueryMetric.getNextCount());
            combinedMetric.setSeekCount(combinedMetric.getSeekCount() + updatedQueryMetric.getSeekCount());
            combinedMetric.setYieldCount(combinedMetric.getYieldCount() + updatedQueryMetric.getYieldCount());
            combinedMetric.setDocRanges(combinedMetric.getDocRanges() + updatedQueryMetric.getDocRanges());
            combinedMetric.setFiRanges(combinedMetric.getFiRanges() + updatedQueryMetric.getFiRanges());
        } else {
            combinedMetric.setSourceCount(updatedQueryMetric.getSourceCount());
            combinedMetric.setNextCount(updatedQueryMetric.getNextCount());
            combinedMetric.setSeekCount(updatedQueryMetric.getSeekCount());
            combinedMetric.setYieldCount(updatedQueryMetric.getYieldCount());
            combinedMetric.setDocRanges(updatedQueryMetric.getDocRanges());
            combinedMetric.setFiRanges(updatedQueryMetric.getFiRanges());
        }
        // only update once
        if (combinedMetric.getPlan() == null && updatedQueryMetric.getPlan() != null) {
            combinedMetric.setPlan(updatedQueryMetric.getPlan());
        }
        // only update once
        if (combinedMetric.getPredictions() == null && updatedQueryMetric.getPredictions() != null) {
            combinedMetric.setPredictions(updatedQueryMetric.getPredictions());
        }
        // use the max numUpdates
        combinedMetric.setNumUpdates(Math.max(combinedMetric.getNumUpdates(), updatedQueryMetric.getNumUpdates()));
    }
    log.trace("Combined metrics cached: " + cachedQueryMetric + " updated: " + updatedQueryMetric + " combined: " + combinedMetric);
    return combinedMetric;
}

Example 9 with PageMetric

use of datawave.microservice.querymetric.BaseQueryMetric.PageMetric in project datawave-query-metric-service by NationalSecurityAgency.

the class QueryMetricCombiner method combinePageMetrics.

protected PageMetric combinePageMetrics(PageMetric updated, PageMetric stored) {
    if (stored == null) {
        return updated;
    }
    String updatedUuid = updated.getPageUuid();
    String storedUuid = stored.getPageUuid();
    if (updatedUuid != null && storedUuid != null && !updatedUuid.equals(storedUuid)) {
        throw new IllegalStateException("can not combine page metrics with different pageUuids: " + "updated:" + updated.getPageUuid() + " stored:" + stored.getPageUuid());
    }
    PageMetric pm = new PageMetric(stored);
    if (pm.getHost() == null) {
        pm.setHost(updated.getHost());
    }
    if (pm.getPagesize() == 0) {
        pm.setPagesize(updated.getPagesize());
    }
    if (pm.getReturnTime() == -1) {
        pm.setReturnTime(updated.getReturnTime());
    }
    if (pm.getCallTime() == -1) {
        pm.setCallTime(updated.getCallTime());
    }
    if (pm.getSerializationTime() == -1) {
        pm.setSerializationTime(updated.getSerializationTime());
    }
    if (pm.getBytesWritten() == -1) {
        pm.setBytesWritten(updated.getBytesWritten());
    }
    if (pm.getPageRequested() == 0) {
        pm.setPageRequested(updated.getPageRequested());
    }
    if (pm.getPageReturned() == 0) {
        pm.setPageReturned(updated.getPageReturned());
    }
    if (pm.getLoginTime() == -1) {
        pm.setLoginTime(updated.getLoginTime());
    }
    return pm;
}

Example 10 with PageMetric

use of datawave.microservice.querymetric.BaseQueryMetric.PageMetric in project datawave-query-metric-service by NationalSecurityAgency.

the class QueryMetricCombiner method getLastPageNumber.

public long getLastPageNumber(BaseQueryMetric m) {
    long lastPage = 0;
    List<PageMetric> pageMetrics = m.getPageTimes();
    for (PageMetric pm : pageMetrics) {
        if (lastPage == 0 || pm.getPageNumber() > lastPage) {
            lastPage = pm.getPageNumber();
        }
    }
    return lastPage;
}