Examples with TimeSpec com.linkedin.thirdeye.api.TimeSpec used on opensource projects

Example 21 with TimeSpec

use of com.linkedin.thirdeye.api.TimeSpec in project pinot by linkedin.

the class AnomalyResource method createAnomalyFunction.

// Add anomaly function
@POST
@Path("/anomaly-function/create")
public Response createAnomalyFunction(@NotNull @QueryParam("dataset") String dataset, @NotNull @QueryParam("functionName") String functionName, @NotNull @QueryParam("metric") String metric, @NotNull @QueryParam("metricFunction") String metric_function, @QueryParam("type") String type, @NotNull @QueryParam("windowSize") String windowSize, @NotNull @QueryParam("windowUnit") String windowUnit, @QueryParam("windowDelay") String windowDelay, @QueryParam("cron") String cron, @QueryParam("windowDelayUnit") String windowDelayUnit, @QueryParam("exploreDimension") String exploreDimensions, @QueryParam("filters") String filters, @NotNull @QueryParam("properties") String properties, @QueryParam("isActive") boolean isActive) throws Exception {
    if (StringUtils.isEmpty(dataset) || StringUtils.isEmpty(functionName) || StringUtils.isEmpty(metric) || StringUtils.isEmpty(windowSize) || StringUtils.isEmpty(windowUnit) || StringUtils.isEmpty(properties)) {
        throw new UnsupportedOperationException("Received null for one of the mandatory params: " + "dataset " + dataset + ", functionName " + functionName + ", metric " + metric + ", windowSize " + windowSize + ", windowUnit " + windowUnit + ", properties" + properties);
    }
    DatasetConfigDTO datasetConfig = DAO_REGISTRY.getDatasetConfigDAO().findByDataset(dataset);
    TimeSpec timespec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
    TimeGranularity dataGranularity = timespec.getDataGranularity();
    AnomalyFunctionDTO anomalyFunctionSpec = new AnomalyFunctionDTO();
    anomalyFunctionSpec.setActive(isActive);
    anomalyFunctionSpec.setMetricFunction(MetricAggFunction.valueOf(metric_function));
    anomalyFunctionSpec.setCollection(dataset);
    anomalyFunctionSpec.setFunctionName(functionName);
    anomalyFunctionSpec.setTopicMetric(metric);
    anomalyFunctionSpec.setMetrics(Arrays.asList(metric));
    if (StringUtils.isEmpty(type)) {
        type = DEFAULT_FUNCTION_TYPE;
    }
    anomalyFunctionSpec.setType(type);
    anomalyFunctionSpec.setWindowSize(Integer.valueOf(windowSize));
    anomalyFunctionSpec.setWindowUnit(TimeUnit.valueOf(windowUnit));
    // Setting window delay time / unit
    TimeUnit dataGranularityUnit = dataGranularity.getUnit();
    // default window delay time = 10 hours
    int windowDelayTime = 10;
    TimeUnit windowDelayTimeUnit = TimeUnit.HOURS;
    if (dataGranularityUnit.equals(TimeUnit.MINUTES) || dataGranularityUnit.equals(TimeUnit.HOURS)) {
        windowDelayTime = 4;
    }
    anomalyFunctionSpec.setWindowDelayUnit(windowDelayTimeUnit);
    anomalyFunctionSpec.setWindowDelay(windowDelayTime);
    // bucket size and unit are defaulted to the collection granularity
    anomalyFunctionSpec.setBucketSize(dataGranularity.getSize());
    anomalyFunctionSpec.setBucketUnit(dataGranularity.getUnit());
    if (StringUtils.isNotEmpty(exploreDimensions)) {
        anomalyFunctionSpec.setExploreDimensions(getDimensions(dataset, exploreDimensions));
    }
    if (!StringUtils.isBlank(filters)) {
        filters = URLDecoder.decode(filters, UTF8);
        String filterString = ThirdEyeUtils.getSortedFiltersFromJson(filters);
        anomalyFunctionSpec.setFilters(filterString);
    }
    anomalyFunctionSpec.setProperties(properties);
    if (StringUtils.isEmpty(cron)) {
        cron = DEFAULT_CRON;
    } else {
        // validate cron
        if (!CronExpression.isValidExpression(cron)) {
            throw new IllegalArgumentException("Invalid cron expression for cron : " + cron);
        }
    }
    anomalyFunctionSpec.setCron(cron);
    Long id = anomalyFunctionDAO.save(anomalyFunctionSpec);
    return Response.ok(id).build();
}

Example 22 with TimeSpec

use of com.linkedin.thirdeye.api.TimeSpec in project pinot by linkedin.

the class DashboardResource method getTimeSeriesData.

@GET
@Path(value = "/data/timeseries")
@Produces(MediaType.APPLICATION_JSON)
public String getTimeSeriesData(@QueryParam("dataset") String collection, @QueryParam("filters") String filterJson, @QueryParam("timeZone") @DefaultValue(DEFAULT_TIMEZONE_ID) String timeZone, @QueryParam("currentStart") Long start, @QueryParam("currentEnd") Long end, @QueryParam("aggTimeGranularity") String aggTimeGranularity, @QueryParam("metrics") String metricsJson, @QueryParam("dimensions") String groupByDimensions) throws Exception {
    TimeSeriesRequest request = new TimeSeriesRequest();
    request.setCollectionName(collection);
    // See {@link #getDashboardData} for the reason that the start and end time are stored in a
    // DateTime object with data's timezone.
    DateTimeZone timeZoneForCollection = Utils.getDataTimeZone(collection);
    request.setStart(new DateTime(start, timeZoneForCollection));
    request.setEnd(new DateTime(end, timeZoneForCollection));
    if (groupByDimensions != null && !groupByDimensions.isEmpty()) {
        request.setGroupByDimensions(Arrays.asList(groupByDimensions.trim().split(",")));
    }
    if (filterJson != null && !filterJson.isEmpty()) {
        filterJson = URLDecoder.decode(filterJson, "UTF-8");
        request.setFilterSet(ThirdEyeUtils.convertToMultiMap(filterJson));
    }
    List<MetricExpression> metricExpressions = Utils.convertToMetricExpressions(metricsJson, MetricAggFunction.SUM, collection);
    request.setMetricExpressions(metricExpressions);
    request.setAggregationTimeGranularity(Utils.getAggregationTimeGranularity(aggTimeGranularity, collection));
    DatasetConfigDTO datasetConfig = CACHE_REGISTRY_INSTANCE.getDatasetConfigCache().get(collection);
    TimeSpec timespec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
    if (!request.getAggregationTimeGranularity().getUnit().equals(TimeUnit.DAYS) || !StringUtils.isBlank(timespec.getFormat())) {
        request.setEndDateInclusive(true);
    }
    TimeSeriesHandler handler = new TimeSeriesHandler(queryCache);
    String jsonResponse = "";
    try {
        TimeSeriesResponse response = handler.handle(request);
        JSONObject timeseriesMap = new JSONObject();
        JSONArray timeValueArray = new JSONArray();
        TreeSet<String> keys = new TreeSet<>();
        TreeSet<Long> times = new TreeSet<>();
        for (int i = 0; i < response.getNumRows(); i++) {
            TimeSeriesRow timeSeriesRow = response.getRow(i);
            times.add(timeSeriesRow.getStart());
        }
        for (Long time : times) {
            timeValueArray.put(time);
        }
        timeseriesMap.put("time", timeValueArray);
        for (int i = 0; i < response.getNumRows(); i++) {
            TimeSeriesRow timeSeriesRow = response.getRow(i);
            for (TimeSeriesMetric metricTimeSeries : timeSeriesRow.getMetrics()) {
                String key = metricTimeSeries.getMetricName();
                if (timeSeriesRow.getDimensionNames() != null && timeSeriesRow.getDimensionNames().size() > 0) {
                    StringBuilder sb = new StringBuilder(key);
                    for (int idx = 0; idx < timeSeriesRow.getDimensionNames().size(); ++idx) {
                        sb.append("||").append(timeSeriesRow.getDimensionNames().get(idx));
                        sb.append("|").append(timeSeriesRow.getDimensionValues().get(idx));
                    }
                    key = sb.toString();
                }
                JSONArray valueArray;
                if (!timeseriesMap.has(key)) {
                    valueArray = new JSONArray();
                    timeseriesMap.put(key, valueArray);
                    keys.add(key);
                } else {
                    valueArray = timeseriesMap.getJSONArray(key);
                }
                valueArray.put(metricTimeSeries.getValue());
            }
        }
        JSONObject summaryMap = new JSONObject();
        summaryMap.put("currentStart", start);
        summaryMap.put("currentEnd", end);
        JSONObject jsonResponseObject = new JSONObject();
        jsonResponseObject.put("timeSeriesData", timeseriesMap);
        jsonResponseObject.put("keys", new JSONArray(keys));
        jsonResponseObject.put("summary", summaryMap);
        jsonResponse = jsonResponseObject.toString();
    } catch (Exception e) {
        throw e;
    }
    LOG.info("Response:{}", jsonResponse);
    return jsonResponse;
}

Example 23 with TimeSpec

use of com.linkedin.thirdeye.api.TimeSpec in project pinot by linkedin.

the class DashboardResource method getMaxTime.

@GET
@Path(value = "/data/info")
@Produces(MediaType.APPLICATION_JSON)
public String getMaxTime(@QueryParam("dataset") String collection) {
    String collectionInfo = null;
    try {
        HashMap<String, String> map = new HashMap<>();
        long maxDataTime = collectionMaxDataTimeCache.get(collection);
        DatasetConfigDTO datasetConfig = CACHE_REGISTRY_INSTANCE.getDatasetConfigCache().get(collection);
        TimeSpec timespec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
        TimeGranularity dataGranularity = timespec.getDataGranularity();
        map.put("maxTime", "" + maxDataTime);
        map.put("dataGranularity", dataGranularity.getUnit().toString());
        List<MetricConfigDTO> metricConfigs = metricConfigDAO.findActiveByDataset(collection);
        List<String> inverseMetrics = new ArrayList<>();
        for (MetricConfigDTO metricConfig : metricConfigs) {
            if (metricConfig.isInverseMetric()) {
                inverseMetrics.add(metricConfig.getName());
            }
        }
        if (CollectionUtils.isNotEmpty(inverseMetrics)) {
            map.put("invertColorMetrics", Joiner.on(",").join(inverseMetrics));
        }
        collectionInfo = OBJECT_MAPPER.writeValueAsString(map);
    } catch (Exception e) {
        LOG.error("Error while fetching info for collection: " + collection, e);
    }
    return collectionInfo;
}

Example 24 with TimeSpec

use of com.linkedin.thirdeye.api.TimeSpec in project pinot by linkedin.

the class EmailHelper method writeTimeSeriesChart.

public static String writeTimeSeriesChart(final EmailConfigurationDTO config, TimeOnTimeComparisonHandler timeOnTimeComparisonHandler, final DateTime now, final DateTime then, final String collection, final Map<RawAnomalyResultDTO, String> anomaliesWithLabels) throws JobExecutionException {
    try {
        int windowSize = config.getWindowSize();
        TimeUnit windowUnit = config.getWindowUnit();
        long windowMillis = windowUnit.toMillis(windowSize);
        // TODO provide a way for email reports to specify desired graph granularity.
        DatasetConfigManager datasetConfigDAO = DAO_REGISTRY.getDatasetConfigDAO();
        DatasetConfigDTO datasetConfig = datasetConfigDAO.findByDataset(collection);
        TimeSpec timespec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
        TimeGranularity dataGranularity = timespec.getDataGranularity();
        TimeOnTimeComparisonResponse chartData = getData(timeOnTimeComparisonHandler, config, then, now, WEEK_MILLIS, dataGranularity);
        AnomalyGraphGenerator anomalyGraphGenerator = AnomalyGraphGenerator.getInstance();
        JFreeChart chart = anomalyGraphGenerator.createChart(chartData, dataGranularity, windowMillis, anomaliesWithLabels);
        String chartFilePath = EMAIL_REPORT_CHART_PREFIX + config.getId() + PNG;
        LOG.info("Writing chart to {}", chartFilePath);
        anomalyGraphGenerator.writeChartToFile(chart, chartFilePath);
        return chartFilePath;
    } catch (Exception e) {
        throw new JobExecutionException(e);
    }
}

Example 25 with TimeSpec

use of com.linkedin.thirdeye.api.TimeSpec in project pinot by linkedin.

the class DetectionJobSchedulerUtils method getBucketSizeInMSForDataset.

/**
   * get bucket size in millis, according to data granularity of dataset
   * Bucket size are 1 HOUR for hourly, 1 DAY for daily
   * For MINUTE level data, bucket size is calculated based on anomaly function frequency
   * @param timeSpec
   * @return
   */
public static long getBucketSizeInMSForDataset(DatasetConfigDTO datasetConfig, AnomalyFunctionDTO anomalyFunction) {
    long bucketMillis = 0;
    TimeSpec timeSpec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
    TimeUnit dataUnit = timeSpec.getDataGranularity().getUnit();
    TimeGranularity functionFrequency = anomalyFunction.getFrequency();
    // Calculate time periods according to the function frequency
    if (dataUnit.equals(TimeUnit.MINUTES) || dataUnit.equals(TimeUnit.MILLISECONDS) || dataUnit.equals(TimeUnit.SECONDS)) {
        if (functionFrequency.getUnit().equals(TimeUnit.MINUTES) && (functionFrequency.getSize() <= 30)) {
            bucketMillis = TimeUnit.MILLISECONDS.convert(functionFrequency.getSize(), TimeUnit.MINUTES);
        } else {
            // default to HOURS
            bucketMillis = TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS);
        }
    } else {
        bucketMillis = TimeUnit.MILLISECONDS.convert(1, dataUnit);
    }
    return bucketMillis;
}