Examples with DatasetConfigDTO com.linkedin.thirdeye.datalayer.dto.DatasetConfigDTO used on opensource projects

Example 6 with DatasetConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.DatasetConfigDTO in project pinot by linkedin.

the class AutoLoadPinotMetricsService method run.

public void run() {
    try {
        loadDatasets();
        for (String dataset : allDatasets) {
            LOG.info("Checking dataset {}", dataset);
            Schema schema = allSchemas.get(dataset);
            if (!isIngraphDataset(schema)) {
                DatasetConfigDTO datasetConfig = DAO_REGISTRY.getDatasetConfigDAO().findByDataset(dataset);
                addPinotDataset(dataset, schema, datasetConfig);
            }
        }
    } catch (Exception e) {
        LOG.error("Exception in loading datasets", e);
    }
}

Example 7 with DatasetConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.DatasetConfigDTO in project pinot by linkedin.

the class CollectionsCache method loadCollections.

public void loadCollections() {
    List<String> collections = new ArrayList<>();
    if (StringUtils.isNotBlank(whitelistCollections)) {
        List<String> whitelist = Lists.newArrayList(whitelistCollections.split(","));
        for (String collection : whitelist) {
            DatasetConfigDTO datasetConfig = datasetConfigDAO.findByDataset(collection);
            if (datasetConfig == null || !datasetConfig.isActive()) {
                LOG.info("Skipping collection {} due to missing dataset config or status inactive", collection);
                continue;
            }
            collections.add(collection);
        }
    } else {
        List<DatasetConfigDTO> datasetConfigs = datasetConfigDAO.findActive();
        for (DatasetConfigDTO datasetConfigDTO : datasetConfigs) {
            collections.add(datasetConfigDTO.getDataset());
        }
    }
    LOG.info("Loading collections {}", collections);
    collectionsRef.set(collections);
}

Example 8 with DatasetConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.DatasetConfigDTO in project pinot by linkedin.

the class AnomaliesResource method getTimeSeriesData.

/**
   * Get timeseries for metric
   * @param collection
   * @param filters
   * @param start
   * @param end
   * @param aggTimeGranularity
   * @param metric
   * @return
   * @throws Exception
   */
private JSONObject getTimeSeriesData(String collection, Multimap<String, String> filters, Long start, Long end, String aggTimeGranularity, String metric) throws Exception {
    TimeSeriesRequest request = new TimeSeriesRequest();
    request.setCollectionName(collection);
    DateTimeZone timeZoneForCollection = Utils.getDataTimeZone(collection);
    request.setStart(new DateTime(start, timeZoneForCollection));
    request.setEnd(new DateTime(end, timeZoneForCollection));
    request.setFilterSet(filters);
    List<MetricExpression> metricExpressions = Utils.convertToMetricExpressions(metric, MetricAggFunction.SUM, collection);
    request.setMetricExpressions(metricExpressions);
    request.setAggregationTimeGranularity(Utils.getAggregationTimeGranularity(aggTimeGranularity, collection));
    DatasetConfigDTO datasetConfig = CACHE_REGISTRY.getDatasetConfigCache().get(collection);
    TimeSpec timespec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
    if (!request.getAggregationTimeGranularity().getUnit().equals(TimeUnit.DAYS) || !StringUtils.isBlank(timespec.getFormat())) {
        request.setEndDateInclusive(true);
    }
    TimeSeriesHandler handler = new TimeSeriesHandler(CACHE_REGISTRY.getQueryCache());
    JSONObject jsonResponseObject = new JSONObject();
    try {
        TimeSeriesResponse response = handler.handle(request);
        JSONObject timeseriesMap = new JSONObject();
        JSONArray timeValueArray = new JSONArray();
        TreeSet<String> keys = new TreeSet<>();
        TreeSet<Long> times = new TreeSet<>();
        for (int i = 0; i < response.getNumRows(); i++) {
            TimeSeriesRow timeSeriesRow = response.getRow(i);
            times.add(timeSeriesRow.getStart());
        }
        for (Long time : times) {
            timeValueArray.put(time);
        }
        timeseriesMap.put("time", timeValueArray);
        for (int i = 0; i < response.getNumRows(); i++) {
            TimeSeriesRow timeSeriesRow = response.getRow(i);
            for (TimeSeriesMetric metricTimeSeries : timeSeriesRow.getMetrics()) {
                String key = metricTimeSeries.getMetricName();
                JSONArray valueArray;
                if (!timeseriesMap.has(key)) {
                    valueArray = new JSONArray();
                    timeseriesMap.put(key, valueArray);
                    keys.add(key);
                } else {
                    valueArray = timeseriesMap.getJSONArray(key);
                }
                valueArray.put(metricTimeSeries.getValue());
            }
        }
        JSONObject summaryMap = new JSONObject();
        summaryMap.put("currentStart", start);
        summaryMap.put("currentEnd", end);
        jsonResponseObject.put("timeSeriesData", timeseriesMap);
        jsonResponseObject.put("keys", new JSONArray(keys));
        jsonResponseObject.put("summary", summaryMap);
    } catch (Exception e) {
        throw e;
    }
    LOG.info("Response:{}", jsonResponseObject);
    return jsonResponseObject;
}

Example 9 with DatasetConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.DatasetConfigDTO in project pinot by linkedin.

the class DatasetConfigResource method viewDatsetConfig.

@GET
@Path("/list")
@Produces(MediaType.APPLICATION_JSON)
public String viewDatsetConfig(@DefaultValue("0") @QueryParam("jtStartIndex") int jtStartIndex, @DefaultValue("100") @QueryParam("jtPageSize") int jtPageSize) {
    List<DatasetConfigDTO> datasetConfigDTOs = datasetConfigDao.findAll();
    List<DatasetConfigDTO> subList = Utils.sublist(datasetConfigDTOs, jtStartIndex, jtPageSize);
    ObjectNode rootNode = JsonResponseUtil.buildResponseJSON(subList);
    return rootNode.toString();
}

Example 10 with DatasetConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.DatasetConfigDTO in project pinot by linkedin.

the class DatasetConfigResource method createDatasetConfig.

@GET
@Path("/create")
public String createDatasetConfig(@QueryParam("dataset") String dataset, @QueryParam("dimensions") String dimensions, @QueryParam("dimensionsHaveNoPreAggregation") String dimensionsHaveNoPreAggregation, @QueryParam("active") boolean active, @QueryParam("additive") boolean additive, @QueryParam("metricAsDimension") boolean metricAsDimension, @QueryParam("metricValuesColumn") String metricValuesColumn, @QueryParam("metricNamesColumn") String metricNamesColumn, @QueryParam("nonAdditiveBucketSize") Integer nonAdditiveBucketSize, @QueryParam("nonAdditiveBucketUnit") String nonAdditiveBucketUnit, @QueryParam("preAggregatedKeyword") String preAggregatedKeyword, @QueryParam("timeColumn") String timeColumn, @QueryParam("timeDuration") Integer timeDuration, @QueryParam("timeFormat") String timeFormat, @QueryParam("timezone") TimeUnit timeUnit, @QueryParam("timezone") String timezone) {
    try {
        DatasetConfigDTO datasetConfigDTO = new DatasetConfigDTO();
        datasetConfigDTO.setDataset(dataset);
        datasetConfigDTO.setDimensions(toList(dimensions));
        if (!Strings.isNullOrEmpty(dimensionsHaveNoPreAggregation)) {
            datasetConfigDTO.setDimensionsHaveNoPreAggregation(toList(dimensionsHaveNoPreAggregation));
        }
        datasetConfigDTO.setActive(active);
        datasetConfigDTO.setAdditive(additive);
        datasetConfigDTO.setMetricAsDimension(metricAsDimension);
        datasetConfigDTO.setMetricNamesColumn(metricNamesColumn);
        datasetConfigDTO.setMetricValuesColumn(metricValuesColumn);
        datasetConfigDTO.setNonAdditiveBucketSize(nonAdditiveBucketSize);
        datasetConfigDTO.setNonAdditiveBucketUnit(nonAdditiveBucketUnit);
        datasetConfigDTO.setPreAggregatedKeyword(preAggregatedKeyword);
        datasetConfigDTO.setTimeColumn(timeColumn);
        datasetConfigDTO.setTimeDuration(timeDuration);
        datasetConfigDTO.setTimeFormat(timeFormat);
        datasetConfigDTO.setTimeUnit(timeUnit);
        datasetConfigDTO.setTimezone(timezone);
        Long id = datasetConfigDao.save(datasetConfigDTO);
        datasetConfigDTO.setId(id);
        return JsonResponseUtil.buildResponseJSON(datasetConfigDTO).toString();
    } catch (Exception e) {
        return JsonResponseUtil.buildErrorResponseJSON("Failed to create dataset:" + dataset).toString();
    }
}