Examples with Pair com.linkedin.pinot.pql.parsers.utils.Pair used on opensource projects

Example 1 with Pair

use of com.linkedin.pinot.pql.parsers.utils.Pair in project pinot by linkedin.

the class AnomalyMergeExecutor method updateMergedAnomalyWeight.

/**
   * Uses function-specific method to re-computes the weight of merged anomaly.
   *
   * @param anomalyMergedResult the merged anomaly to be updated
   * @param mergeConfig the merge configuration that was applied when merge the merged anomaly
   * @throws Exception if error occurs when retrieving the time series for calculating the weight
   */
private void updateMergedAnomalyWeight(MergedAnomalyResultDTO anomalyMergedResult, AnomalyMergeConfig mergeConfig) throws Exception {
    AnomalyFunctionDTO anomalyFunctionSpec = anomalyMergedResult.getFunction();
    BaseAnomalyFunction anomalyFunction = anomalyFunctionFactory.fromSpec(anomalyFunctionSpec);
    List<Pair<Long, Long>> startEndTimeRanges = anomalyFunction.getDataRangeIntervals(anomalyMergedResult.getStartTime(), anomalyMergedResult.getEndTime());
    TimeGranularity timeGranularity = new TimeGranularity(anomalyFunctionSpec.getBucketSize(), anomalyFunctionSpec.getBucketUnit());
    MetricTimeSeries metricTimeSeries = TimeSeriesUtil.getTimeSeriesByDimension(anomalyFunctionSpec, startEndTimeRanges, anomalyMergedResult.getDimensions(), timeGranularity, false);
    if (metricTimeSeries != null) {
        DateTime windowStart = new DateTime(anomalyMergedResult.getStartTime());
        DateTime windowEnd = new DateTime(anomalyMergedResult.getEndTime());
        List<MergedAnomalyResultDTO> knownAnomalies = Collections.emptyList();
        // Retrieve history merged anomalies
        if (anomalyFunction.useHistoryAnomaly()) {
            switch(mergeConfig.getMergeStrategy()) {
                case FUNCTION:
                    knownAnomalies = getHistoryMergedAnomalies(anomalyFunction, windowStart.getMillis(), windowEnd.getMillis());
                    break;
                case FUNCTION_DIMENSIONS:
                    knownAnomalies = getHistoryMergedAnomalies(anomalyFunction, windowStart.getMillis(), windowEnd.getMillis(), anomalyMergedResult.getDimensions());
                    break;
                default:
                    throw new IllegalArgumentException("Merge strategy " + mergeConfig.getMergeStrategy() + " not supported");
            }
            if (knownAnomalies.size() > 0) {
                LOG.info("Found {} history anomalies for computing the weight of current merged anomaly.", knownAnomalies.size());
                LOG.info("Checking if any known anomalies overlap with the monitoring window of anomaly detection, which could result in unwanted holes in current values.");
                AnomalyUtils.logAnomaliesOverlapWithWindow(windowStart, windowEnd, knownAnomalies);
            }
        }
        // Transform Time Series
        List<ScalingFactor> scalingFactors = OverrideConfigHelper.getTimeSeriesScalingFactors(overrideConfigDAO, anomalyFunctionSpec.getCollection(), anomalyFunctionSpec.getTopicMetric(), anomalyFunctionSpec.getId(), anomalyFunction.getDataRangeIntervals(windowStart.getMillis(), windowEnd.getMillis()));
        if (CollectionUtils.isNotEmpty(scalingFactors)) {
            Properties properties = anomalyFunction.getProperties();
            MetricTransfer.rescaleMetric(metricTimeSeries, windowStart.getMillis(), scalingFactors, anomalyFunctionSpec.getTopicMetric(), properties);
        }
        anomalyFunction.updateMergedAnomalyInfo(anomalyMergedResult, metricTimeSeries, windowStart, windowEnd, knownAnomalies);
    }
}

Example 2 with Pair

use of com.linkedin.pinot.pql.parsers.utils.Pair in project pinot by linkedin.

the class TimeBasedAnomalyMerger method fetchDataByDimension.

/**
   * Fetch time series, known merged anomalies, and scaling factor for the specified dimension. Note that scaling
   * factor has no dimension information, so all scaling factor in the specified time range will be retrieved.
   *
   * @param windowStartTime the start time for retrieving the data
   * @param windowEndTime the end time for retrieving the data
   * @param dimensions the dimension of the data
   * @param anomalyFunction the anomaly function that produces the anomaly
   * @param mergedResultDAO DAO for merged anomalies
   * @param overrideConfigDAO DAO for override configuration
   * @param endTimeInclusive set to true if the end time should be inclusive; mainly used by the queries from UI
   * @return an anomaly detection input context that contains all the retrieved data
   * @throws Exception if it fails to retrieve time series from DB.
   */
public static AnomalyDetectionInputContext fetchDataByDimension(long windowStartTime, long windowEndTime, DimensionMap dimensions, BaseAnomalyFunction anomalyFunction, MergedAnomalyResultManager mergedResultDAO, OverrideConfigManager overrideConfigDAO, boolean endTimeInclusive) throws Exception {
    AnomalyFunctionDTO functionSpec = anomalyFunction.getSpec();
    List<Pair<Long, Long>> startEndTimeRanges = anomalyFunction.getDataRangeIntervals(windowStartTime, windowEndTime);
    TimeGranularity timeGranularity = new TimeGranularity(functionSpec.getBucketSize(), functionSpec.getBucketUnit());
    AnomalyDetectionInputContext adInputContext = new AnomalyDetectionInputContext();
    // Retrieve Time Series
    MetricTimeSeries metricTimeSeries = TimeSeriesUtil.getTimeSeriesByDimension(functionSpec, startEndTimeRanges, dimensions, timeGranularity, endTimeInclusive);
    Map<DimensionMap, MetricTimeSeries> metricTimeSeriesMap = new HashMap<>();
    metricTimeSeriesMap.put(dimensions, metricTimeSeries);
    adInputContext.setDimensionKeyMetricTimeSeriesMap(metricTimeSeriesMap);
    // Retrieve historical anomaly
    if (anomalyFunction.useHistoryAnomaly()) {
        List<MergedAnomalyResultDTO> knownAnomalies = getBaselineKnownAnomaliesByDimension(anomalyFunction, windowStartTime, windowEndTime, dimensions, mergedResultDAO);
        ListMultimap<DimensionMap, MergedAnomalyResultDTO> mergedAnomalyMap = ArrayListMultimap.create();
        mergedAnomalyMap.putAll(dimensions, knownAnomalies);
        adInputContext.setKnownMergedAnomalies(mergedAnomalyMap);
        if (knownAnomalies.size() > 0) {
            LOG.info("Found {} history anomalies for computing the weight of current merged anomaly.", knownAnomalies.size());
        }
    }
    // Retrieve scaling factor
    List<ScalingFactor> scalingFactors = OverrideConfigHelper.getTimeSeriesScalingFactors(overrideConfigDAO, functionSpec.getCollection(), functionSpec.getTopicMetric(), functionSpec.getId(), anomalyFunction.getDataRangeIntervals(windowStartTime, windowEndTime));
    adInputContext.setScalingFactors(scalingFactors);
    return adInputContext;
}

Example 3 with Pair

use of com.linkedin.pinot.pql.parsers.utils.Pair in project pinot by linkedin.

the class AnomalyFunctionResource method analyze.

@POST
@Path("/analyze")
@Consumes(MediaType.APPLICATION_JSON)
public Response analyze(AnomalyFunctionDTO anomalyFunctionSpec, @QueryParam("startTime") Long startTime, @QueryParam("endTime") Long endTime) throws Exception {
    // TODO: replace this with Job/Task framework and job tracker page
    BaseAnomalyFunction anomalyFunction = anomalyFunctionFactory.fromSpec(anomalyFunctionSpec);
    List<Pair<Long, Long>> startEndTimeRanges = anomalyFunction.getDataRangeIntervals(startTime, endTime);
    Map<DimensionKey, MetricTimeSeries> dimensionKeyMetricTimeSeriesMap = TimeSeriesUtil.getTimeSeriesForAnomalyDetection(anomalyFunctionSpec, startEndTimeRanges);
    List<RawAnomalyResultDTO> anomalyResults = new ArrayList<>();
    List<RawAnomalyResultDTO> results = new ArrayList<>();
    List<String> collectionDimensions = DAO_REGISTRY.getDatasetConfigDAO().findByDataset(anomalyFunctionSpec.getCollection()).getDimensions();
    for (Map.Entry<DimensionKey, MetricTimeSeries> entry : dimensionKeyMetricTimeSeriesMap.entrySet()) {
        DimensionKey dimensionKey = entry.getKey();
        DimensionMap dimensionMap = DimensionMap.fromDimensionKey(dimensionKey, collectionDimensions);
        if (entry.getValue().getTimeWindowSet().size() < 2) {
            LOG.warn("Insufficient data for {} to run anomaly detection function", dimensionMap);
            continue;
        }
        try {
            // Run algorithm
            MetricTimeSeries metricTimeSeries = entry.getValue();
            LOG.info("Analyzing anomaly function with dimensionKey: {}, windowStart: {}, windowEnd: {}", dimensionMap, startTime, endTime);
            List<RawAnomalyResultDTO> resultsOfAnEntry = anomalyFunction.analyze(dimensionMap, metricTimeSeries, new DateTime(startTime), new DateTime(endTime), new ArrayList<>());
            if (resultsOfAnEntry.size() != 0) {
                results.addAll(resultsOfAnEntry);
            }
            LOG.info("{} has {} anomalies in window {} to {}", dimensionMap, resultsOfAnEntry.size(), new DateTime(startTime), new DateTime(endTime));
        } catch (Exception e) {
            LOG.error("Could not compute for {}", dimensionMap, e);
        }
    }
    if (results.size() > 0) {
        List<RawAnomalyResultDTO> validResults = new ArrayList<>();
        for (RawAnomalyResultDTO anomaly : results) {
            if (!anomaly.isDataMissing()) {
                LOG.info("Found anomaly, sev [{}] start [{}] end [{}]", anomaly.getWeight(), new DateTime(anomaly.getStartTime()), new DateTime(anomaly.getEndTime()));
                validResults.add(anomaly);
            }
        }
        anomalyResults.addAll(validResults);
    }
    return Response.ok(anomalyResults).build();
}

Example 4 with Pair

use of com.linkedin.pinot.pql.parsers.utils.Pair in project pinot by linkedin.

the class AnomaliesResource method getAnomalyDataCompareResults.

@GET
@Path("/{anomalyId}")
public AnomalyDataCompare.Response getAnomalyDataCompareResults(@PathParam("anomalyId") Long anomalyId) {
    MergedAnomalyResultDTO anomaly = mergedAnomalyResultDAO.findById(anomalyId);
    if (anomaly == null) {
        LOG.error("Anomaly not found with id " + anomalyId);
        throw new IllegalArgumentException("Anomaly not found with id " + anomalyId);
    }
    AnomalyDataCompare.Response response = new AnomalyDataCompare.Response();
    response.setCurrentStart(anomaly.getStartTime());
    response.setCurrenEnd(anomaly.getEndTime());
    try {
        DatasetConfigDTO dataset = datasetConfigDAO.findByDataset(anomaly.getCollection());
        TimeGranularity granularity = new TimeGranularity(dataset.getTimeDuration(), dataset.getTimeUnit());
        // Lets compute currentTimeRange
        Pair<Long, Long> currentTmeRange = new Pair<>(anomaly.getStartTime(), anomaly.getEndTime());
        MetricTimeSeries ts = TimeSeriesUtil.getTimeSeriesByDimension(anomaly.getFunction(), Arrays.asList(currentTmeRange), anomaly.getDimensions(), granularity, false);
        double currentVal = getTotalFromTimeSeries(ts, dataset.isAdditive());
        response.setCurrentVal(currentVal);
        for (AlertConfigBean.COMPARE_MODE compareMode : AlertConfigBean.COMPARE_MODE.values()) {
            long baselineOffset = EmailHelper.getBaselineOffset(compareMode);
            Pair<Long, Long> baselineTmeRange = new Pair<>(anomaly.getStartTime() - baselineOffset, anomaly.getEndTime() - baselineOffset);
            MetricTimeSeries baselineTs = TimeSeriesUtil.getTimeSeriesByDimension(anomaly.getFunction(), Arrays.asList(baselineTmeRange), anomaly.getDimensions(), granularity, false);
            AnomalyDataCompare.CompareResult cr = new AnomalyDataCompare.CompareResult();
            double baseLineval = getTotalFromTimeSeries(baselineTs, dataset.isAdditive());
            cr.setBaselineValue(baseLineval);
            cr.setCompareMode(compareMode);
            cr.setChange(calculateChange(currentVal, baseLineval));
            response.getCompareResults().add(cr);
        }
    } catch (Exception e) {
        LOG.error("Error fetching the timeseries data from pinot", e);
        throw new RuntimeException(e);
    }
    return response;
}

Example 5 with Pair

use of com.linkedin.pinot.pql.parsers.utils.Pair in project pinot by linkedin.

the class BackwardAnomalyFunctionUtils method toBackwardCompatibleDataRanges.

public static List<Pair<Long, Long>> toBackwardCompatibleDataRanges(List<Interval> timeSeriesIntervals) {
    List<Pair<Long, Long>> dataRanges = new ArrayList<>();
    for (Interval interval : timeSeriesIntervals) {
        Pair<Long, Long> dataRange = new Pair<>(interval.getStartMillis(), interval.getEndMillis());
        dataRanges.add(dataRange);
    }
    return dataRanges;
}