Examples with DashboardConfigDTO com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO used on opensource projects

Example 16 with DashboardConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO in project pinot by linkedin.

the class AutoLoadPinotMetricsService method checkMetricAsDimensionDataset.

private void checkMetricAsDimensionDataset(DatasetConfigDTO datasetConfigDTO, Schema schema) {
    String dataset = datasetConfigDTO.getDataset();
    String metricNamesColumn = datasetConfigDTO.getMetricNamesColumn();
    String metricValuesColumn = datasetConfigDTO.getMetricValuesColumn();
    FieldSpec metricValuesColumnFieldSpec = schema.getFieldSpecFor(metricValuesColumn);
    String dashboardName = ThirdEyeUtils.getDefaultDashboardName(dataset);
    // remove metricNamesColumn from dimensions if exists
    List<String> dimensions = datasetConfigDTO.getDimensions();
    if (dimensions.contains(metricNamesColumn)) {
        dimensions.removeAll(Lists.newArrayList(metricNamesColumn));
        datasetConfigDTO.setDimensions(dimensions);
        DAO_REGISTRY.getDatasetConfigDAO().update(datasetConfigDTO);
    }
    // remove metricValuesColumn from metrics if exists
    MetricConfigDTO metricConfigDTO = DAO_REGISTRY.getMetricConfigDAO().findByMetricAndDataset(metricValuesColumn, dataset);
    if (metricConfigDTO != null) {
        Long metricId = metricConfigDTO.getId();
        DAO_REGISTRY.getMetricConfigDAO().delete(metricConfigDTO);
        // remove metricValuesColumn id from default dashboard
        DashboardConfigDTO dashboardConfig = DAO_REGISTRY.getDashboardConfigDAO().findByName(dashboardName);
        List<Long> dashboardMetricIds = dashboardConfig.getMetricIds();
        dashboardMetricIds.removeAll(Lists.newArrayList(metricId));
        LOG.info("Updating dashboard config for {}", dashboardName);
        DAO_REGISTRY.getDashboardConfigDAO().update(dashboardConfig);
    }
    if (datasetConfigDTO.isAutoDiscoverMetrics()) {
        // query pinot to fetch distinct metricNamesColumn
        List<String> allDistinctMetricNames = fetchMetricAsADimensionMetrics(dataset, metricNamesColumn);
        // create metrics for these metric names, if they dont exist
        List<MetricConfigDTO> existingMetricConfigs = DAO_REGISTRY.getMetricConfigDAO().findByDataset(dataset);
        List<String> existingMetricNames = Lists.newArrayList();
        for (MetricConfigDTO existingMetricConfig : existingMetricConfigs) {
            existingMetricNames.add(existingMetricConfig.getName());
        }
        allDistinctMetricNames.removeAll(existingMetricNames);
        for (String metricName : allDistinctMetricNames) {
            LOG.info("Creating metric config for {}", metricName);
            MetricFieldSpec metricFieldSpec = new MetricFieldSpec(metricName, metricValuesColumnFieldSpec.getDataType());
            MetricConfigDTO metricConfig = ConfigGenerator.generateMetricConfig(metricFieldSpec, dataset);
            DAO_REGISTRY.getMetricConfigDAO().save(metricConfig);
        }
        // Add metrics to default dashboard
        List<Long> allMetricIds = ConfigGenerator.getMetricIdsFromMetricConfigs(DAO_REGISTRY.getMetricConfigDAO().findByDataset(dataset));
        DashboardConfigDTO dashboardConfig = DAO_REGISTRY.getDashboardConfigDAO().findByName(dashboardName);
        dashboardConfig.setMetricIds(allMetricIds);
        LOG.info("Creating dashboard config for {}", dashboardName);
        DAO_REGISTRY.getDashboardConfigDAO().update(dashboardConfig);
    }
}

Example 17 with DashboardConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO in project pinot by linkedin.

the class AutoloadPinotMetricsServiceTest method testAddNewDataset.

@Test
public void testAddNewDataset() throws Exception {
    testAutoLoadPinotMetricsService = new AutoLoadPinotMetricsService();
    schema = Schema.fromInputSteam(ClassLoader.getSystemResourceAsStream("sample-pinot-schema.json"));
    testAutoLoadPinotMetricsService.addPinotDataset(dataset, schema, datasetConfig);
    Assert.assertEquals(datasetConfigDAO.findAll().size(), 1);
    datasetConfig = datasetConfigDAO.findByDataset(dataset);
    Assert.assertEquals(datasetConfig.getDataset(), dataset);
    Assert.assertEquals(datasetConfig.getDimensions(), schema.getDimensionNames());
    Assert.assertEquals(datasetConfig.getTimeColumn(), schema.getTimeColumnName());
    TimeGranularitySpec timeGranularitySpec = schema.getTimeFieldSpec().getOutgoingGranularitySpec();
    Assert.assertEquals(datasetConfig.getTimeUnit(), timeGranularitySpec.getTimeType());
    Assert.assertEquals(datasetConfig.getTimeDuration(), new Integer(timeGranularitySpec.getTimeUnitSize()));
    Assert.assertEquals(datasetConfig.getTimeFormat(), timeGranularitySpec.getTimeFormat());
    Assert.assertEquals(datasetConfig.getTimezone(), "UTC");
    List<MetricConfigDTO> metricConfigs = metricConfigDAO.findByDataset(dataset);
    List<String> schemaMetricNames = schema.getMetricNames();
    List<Long> metricIds = new ArrayList<>();
    Assert.assertEquals(metricConfigs.size(), schemaMetricNames.size());
    for (MetricConfigDTO metricConfig : metricConfigs) {
        Assert.assertTrue(schemaMetricNames.contains(metricConfig.getName()));
        metricIds.add(metricConfig.getId());
    }
    DashboardConfigDTO dashboardConfig = dashboardConfigDAO.findByName(DashboardConfigBean.DEFAULT_DASHBOARD_PREFIX + dataset);
    Assert.assertEquals(dashboardConfig.getMetricIds(), metricIds);
}

Example 18 with DashboardConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO in project pinot by linkedin.

the class AutoloadPinotMetricsServiceTest method testRefreshDataset.

@Test(dependsOnMethods = { "testAddNewDataset" })
public void testRefreshDataset() throws Exception {
    DimensionFieldSpec dimensionFieldSpec = new DimensionFieldSpec("newDimension", DataType.STRING, true);
    schema.addField(dimensionFieldSpec);
    testAutoLoadPinotMetricsService.addPinotDataset(dataset, schema, datasetConfig);
    Assert.assertEquals(datasetConfigDAO.findAll().size(), 1);
    DatasetConfigDTO newDatasetConfig1 = datasetConfigDAO.findByDataset(dataset);
    Assert.assertEquals(newDatasetConfig1.getDataset(), dataset);
    Assert.assertEquals(Sets.newHashSet(newDatasetConfig1.getDimensions()), Sets.newHashSet(schema.getDimensionNames()));
    MetricFieldSpec metricFieldSpec = new MetricFieldSpec("newMetric", DataType.LONG);
    schema.addField(metricFieldSpec);
    testAutoLoadPinotMetricsService.addPinotDataset(dataset, schema, newDatasetConfig1);
    Assert.assertEquals(datasetConfigDAO.findAll().size(), 1);
    List<MetricConfigDTO> metricConfigs = metricConfigDAO.findByDataset(dataset);
    List<String> schemaMetricNames = schema.getMetricNames();
    List<Long> metricIds = new ArrayList<>();
    Assert.assertEquals(metricConfigs.size(), schemaMetricNames.size());
    for (MetricConfigDTO metricConfig : metricConfigs) {
        Assert.assertTrue(schemaMetricNames.contains(metricConfig.getName()));
        metricIds.add(metricConfig.getId());
    }
    DashboardConfigDTO dashboardConfig = dashboardConfigDAO.findByName(DashboardConfigBean.DEFAULT_DASHBOARD_PREFIX + dataset);
    Assert.assertEquals(dashboardConfig.getMetricIds(), metricIds);
}