Examples with AnomalyFunctionDTO com.linkedin.thirdeye.datalayer.dto.AnomalyFunctionDTO used on opensource projects

Example 21 with AnomalyFunctionDTO

use of com.linkedin.thirdeye.datalayer.dto.AnomalyFunctionDTO in project pinot by linkedin.

the class TestDetectionJobSchedulerUtils method testGetNewEntriesForDetectionSchedulerDaily.

@Test
public void testGetNewEntriesForDetectionSchedulerDaily() throws Exception {
    DatasetConfigDTO datasetConfig = new DatasetConfigDTO();
    datasetConfig.setTimeColumn("Date");
    datasetConfig.setTimeUnit(TimeUnit.DAYS);
    datasetConfig.setTimeDuration(1);
    DateTimeZone dateTimeZone = DateTimeZone.UTC;
    AnomalyFunctionDTO anomalyFunction = new AnomalyFunctionDTO();
    DateTimeFormatter dateTimeFormatter = DetectionJobSchedulerUtils.getDateTimeFormatterForDataset(datasetConfig, dateTimeZone);
    String currentDateTimeString = "201702140337";
    String currentDateTimeStringRounded = "20170214";
    DateTime currentDateTime = minuteDateTimeFormatter.parseDateTime(currentDateTimeString);
    DateTime currentDateTimeRounded = dateTimeFormatter.parseDateTime(currentDateTimeStringRounded);
    DetectionStatusDTO lastEntryForFunction = null;
    // null last entry
    Map<String, Long> newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 1);
    Assert.assertEquals(newEntries.get(currentDateTimeStringRounded), new Long(currentDateTimeRounded.getMillis()));
    // last entry same as current time
    lastEntryForFunction = new DetectionStatusDTO();
    lastEntryForFunction.setDateToCheckInSDF(currentDateTimeStringRounded);
    lastEntryForFunction.setDateToCheckInMS(currentDateTimeRounded.getMillis());
    newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 0);
    // last entry 1 day before current time
    String lastEntryDateTimeString = "20170213";
    DateTime lastEntryDateTime = dateTimeFormatter.parseDateTime(lastEntryDateTimeString);
    lastEntryForFunction = new DetectionStatusDTO();
    lastEntryForFunction.setDateToCheckInSDF(lastEntryDateTimeString);
    lastEntryForFunction.setDateToCheckInMS(lastEntryDateTime.getMillis());
    newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 1);
    Assert.assertEquals(newEntries.get(currentDateTimeStringRounded), new Long(currentDateTimeRounded.getMillis()));
    // last entry 3 days before current time
    lastEntryDateTimeString = "20170211";
    lastEntryDateTime = dateTimeFormatter.parseDateTime(lastEntryDateTimeString);
    lastEntryForFunction = new DetectionStatusDTO();
    lastEntryForFunction.setDateToCheckInSDF(lastEntryDateTimeString);
    lastEntryForFunction.setDateToCheckInMS(lastEntryDateTime.getMillis());
    newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 3);
    Assert.assertNotNull(newEntries.get("20170212"));
    Assert.assertNotNull(newEntries.get("20170213"));
    Assert.assertNotNull(newEntries.get("20170214"));
    Assert.assertEquals(newEntries.get(currentDateTimeStringRounded), new Long(currentDateTimeRounded.getMillis()));
}

Example 22 with AnomalyFunctionDTO

use of com.linkedin.thirdeye.datalayer.dto.AnomalyFunctionDTO in project pinot by linkedin.

the class TestDetectionJobSchedulerUtils method testGetNewEntriesForDetectionSchedulerHourly.

@Test
public void testGetNewEntriesForDetectionSchedulerHourly() throws Exception {
    DatasetConfigDTO datasetConfig = new DatasetConfigDTO();
    datasetConfig.setTimeColumn("Date");
    datasetConfig.setTimeUnit(TimeUnit.HOURS);
    datasetConfig.setTimeDuration(1);
    DateTimeZone dateTimeZone = DateTimeZone.UTC;
    AnomalyFunctionDTO anomalyFunction = new AnomalyFunctionDTO();
    DateTimeFormatter dateTimeFormatter = DetectionJobSchedulerUtils.getDateTimeFormatterForDataset(datasetConfig, dateTimeZone);
    String currentDateTimeString = "201702140336";
    String currentDateTimeStringRounded = "2017021403";
    DateTime currentDateTime = minuteDateTimeFormatter.parseDateTime(currentDateTimeString);
    DateTime currentDateTimeRounded = dateTimeFormatter.parseDateTime(currentDateTimeStringRounded);
    DetectionStatusDTO lastEntryForFunction = null;
    // null last entry
    Map<String, Long> newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 1);
    Assert.assertEquals(newEntries.get(currentDateTimeStringRounded), new Long(currentDateTimeRounded.getMillis()));
    // last entry same as current time
    lastEntryForFunction = new DetectionStatusDTO();
    lastEntryForFunction.setDateToCheckInSDF(currentDateTimeStringRounded);
    lastEntryForFunction.setDateToCheckInMS(currentDateTimeRounded.getMillis());
    newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 0);
    // last entry 1 hour before current time
    String lastEntryDateTimeString = "2017021402";
    DateTime lastEntryDateTime = dateTimeFormatter.parseDateTime(lastEntryDateTimeString);
    lastEntryForFunction = new DetectionStatusDTO();
    lastEntryForFunction.setDateToCheckInSDF(lastEntryDateTimeString);
    lastEntryForFunction.setDateToCheckInMS(lastEntryDateTime.getMillis());
    newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 1);
    Assert.assertEquals(newEntries.get(currentDateTimeStringRounded), new Long(currentDateTimeRounded.getMillis()));
    // last entry 3 hours before current time
    lastEntryDateTimeString = "2017021400";
    lastEntryDateTime = dateTimeFormatter.parseDateTime(lastEntryDateTimeString);
    lastEntryForFunction = new DetectionStatusDTO();
    lastEntryForFunction.setDateToCheckInSDF(lastEntryDateTimeString);
    lastEntryForFunction.setDateToCheckInMS(lastEntryDateTime.getMillis());
    newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 3);
    Assert.assertNotNull(newEntries.get("2017021401"));
    Assert.assertNotNull(newEntries.get("2017021402"));
    Assert.assertNotNull(newEntries.get("2017021403"));
    Assert.assertEquals(newEntries.get(currentDateTimeStringRounded), new Long(currentDateTimeRounded.getMillis()));
}

Example 23 with AnomalyFunctionDTO

use of com.linkedin.thirdeye.datalayer.dto.AnomalyFunctionDTO in project pinot by linkedin.

the class TestAnomalyFunctionFactory method specWithType.

// helper to abstract specific implementation details.
private AnomalyFunctionDTO specWithType(String type) {
    AnomalyFunctionDTO spec = new AnomalyFunctionDTO();
    spec.setType(type);
    return spec;
}

Example 24 with AnomalyFunctionDTO

use of com.linkedin.thirdeye.datalayer.dto.AnomalyFunctionDTO in project pinot by linkedin.

the class CloneFunctionAndAutoTuneAlertFilterTool method cloneFunctionsToAutoTune.

public Map<String, String> cloneFunctionsToAutoTune(String Collection, Boolean isRemoveHoliday, String holidayFileName) {
    List<AnomalyFunctionDTO> anomalyFunctionSpecs = anomalyFunctionDAO.findAllByCollection(Collection);
    Map<String, String> clonedFunctionIds = new HashMap<>();
    for (AnomalyFunctionDTO anomalyFunctionSpec : anomalyFunctionSpecs) {
        Long functionId = anomalyFunctionSpec.getId();
        Long clonedFunctionId = cloneFunctionToAutotune(functionId, AUTOTUNE_TAG, IS_CLONE_ANOMALY);
        // remove holiday for cloned functionId
        if (isRemoveHoliday) {
            try {
                Map<Long, Long> holidayMap = readTwoColumnsCSVToMap(holidayFileName);
                for (Map.Entry<Long, Long> pair : holidayMap.entrySet()) {
                    callRemoveMergedAnomalies(clonedFunctionId, pair.getKey(), pair.getValue());
                }
            } catch (Exception e) {
                LOG.warn("Error for holiday removal", e.getMessage());
            }
        }
        clonedFunctionIds.put(String.valueOf(functionId), String.valueOf(clonedFunctionId));
    }
    return clonedFunctionIds;
}

Example 25 with AnomalyFunctionDTO

use of com.linkedin.thirdeye.datalayer.dto.AnomalyFunctionDTO in project pinot by linkedin.

the class CloneFunctionAndAutoTuneAlertFilterTool method evalAlertFilterToEvalNode.

public EvaluationNode evalAlertFilterToEvalNode(Long functionID, Long startTime, Long endTime) throws IOException, JSONException {
    AnomalyFunctionDTO anomalyFunctionSpec = anomalyFunctionDAO.findById(functionID);
    String metricName = anomalyFunctionSpec.getFunctionName();
    String collection = anomalyFunctionSpec.getCollection();
    String metricAlertFilter = (anomalyFunctionSpec.getAlertFilter() == null) ? "null" : anomalyFunctionSpec.getAlertFilter().toString().replaceAll(CSVSEPERATOR, CSVESCAPE);
    Properties alertFilterEvaluations = new Properties();
    String evals = getAlertFilterEvaluation(functionID, startTime, endTime);
    if (evals != null) {
        JSONObject evalJson = new JSONObject(evals);
        Iterator<String> nameltr = evalJson.keys();
        while (nameltr.hasNext()) {
            String key = nameltr.next();
            String eval = evalJson.getString(key);
            alertFilterEvaluations.put(key, eval);
        }
    }
    return new EvaluationNode(functionID, metricName, collection, metricAlertFilter, alertFilterEvaluations);
}