Examples with DateTimeFormatter org.joda.time.format.DateTimeFormatter used on opensource projects

Example 36 with DateTimeFormatter

use of org.joda.time.format.DateTimeFormatter in project pinot by linkedin.

the class DataCompletenessTaskUtils method getDateTimeFormatterForDataset.

/**
   * Get date time formatter according to granularity of dataset
   * This is to store the date in the db, in the correct SDF
   * @param timeSpec
   * @return
   */
public static DateTimeFormatter getDateTimeFormatterForDataset(TimeSpec timeSpec, DateTimeZone zone) {
    String pattern = null;
    TimeUnit unit = timeSpec.getDataGranularity().getUnit();
    switch(unit) {
        case DAYS:
            pattern = DAY_FORMAT;
            break;
        case MINUTES:
            pattern = MINUTE_FORMAT;
            break;
        case HOURS:
        default:
            pattern = HOUR_FORMAT;
            break;
    }
    DateTimeFormatter dateTimeFormatter = DateTimeFormat.forPattern(pattern).withZone(zone);
    return dateTimeFormatter;
}

Example 37 with DateTimeFormatter

use of org.joda.time.format.DateTimeFormatter in project pinot by linkedin.

the class TestDetectionJobSchedulerUtils method testGetNewEntriesForDetectionSchedulerDaily.

@Test
public void testGetNewEntriesForDetectionSchedulerDaily() throws Exception {
    DatasetConfigDTO datasetConfig = new DatasetConfigDTO();
    datasetConfig.setTimeColumn("Date");
    datasetConfig.setTimeUnit(TimeUnit.DAYS);
    datasetConfig.setTimeDuration(1);
    DateTimeZone dateTimeZone = DateTimeZone.UTC;
    AnomalyFunctionDTO anomalyFunction = new AnomalyFunctionDTO();
    DateTimeFormatter dateTimeFormatter = DetectionJobSchedulerUtils.getDateTimeFormatterForDataset(datasetConfig, dateTimeZone);
    String currentDateTimeString = "201702140337";
    String currentDateTimeStringRounded = "20170214";
    DateTime currentDateTime = minuteDateTimeFormatter.parseDateTime(currentDateTimeString);
    DateTime currentDateTimeRounded = dateTimeFormatter.parseDateTime(currentDateTimeStringRounded);
    DetectionStatusDTO lastEntryForFunction = null;
    // null last entry
    Map<String, Long> newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 1);
    Assert.assertEquals(newEntries.get(currentDateTimeStringRounded), new Long(currentDateTimeRounded.getMillis()));
    // last entry same as current time
    lastEntryForFunction = new DetectionStatusDTO();
    lastEntryForFunction.setDateToCheckInSDF(currentDateTimeStringRounded);
    lastEntryForFunction.setDateToCheckInMS(currentDateTimeRounded.getMillis());
    newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 0);
    // last entry 1 day before current time
    String lastEntryDateTimeString = "20170213";
    DateTime lastEntryDateTime = dateTimeFormatter.parseDateTime(lastEntryDateTimeString);
    lastEntryForFunction = new DetectionStatusDTO();
    lastEntryForFunction.setDateToCheckInSDF(lastEntryDateTimeString);
    lastEntryForFunction.setDateToCheckInMS(lastEntryDateTime.getMillis());
    newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 1);
    Assert.assertEquals(newEntries.get(currentDateTimeStringRounded), new Long(currentDateTimeRounded.getMillis()));
    // last entry 3 days before current time
    lastEntryDateTimeString = "20170211";
    lastEntryDateTime = dateTimeFormatter.parseDateTime(lastEntryDateTimeString);
    lastEntryForFunction = new DetectionStatusDTO();
    lastEntryForFunction.setDateToCheckInSDF(lastEntryDateTimeString);
    lastEntryForFunction.setDateToCheckInMS(lastEntryDateTime.getMillis());
    newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 3);
    Assert.assertNotNull(newEntries.get("20170212"));
    Assert.assertNotNull(newEntries.get("20170213"));
    Assert.assertNotNull(newEntries.get("20170214"));
    Assert.assertEquals(newEntries.get(currentDateTimeStringRounded), new Long(currentDateTimeRounded.getMillis()));
}

Example 38 with DateTimeFormatter

use of org.joda.time.format.DateTimeFormatter in project pinot by linkedin.

the class DataCompletenessTaskUtilsTest method testGetDateTimeFormatterForDataset.

@Test
public void testGetDateTimeFormatterForDataset() {
    DateTimeZone zone = DateTimeZone.UTC;
    long dateTimeInMS = new DateTime(2017, 01, 12, 15, 30, zone).getMillis();
    String columnName = "Date";
    // DAYS bucket
    TimeGranularity timeGranularity = new TimeGranularity(1, TimeUnit.DAYS);
    String timeFormat = TimeSpec.SINCE_EPOCH_FORMAT;
    TimeSpec timeSpec = new TimeSpec(columnName, timeGranularity, timeFormat);
    DateTimeFormatter dateTimeFormatter = DataCompletenessTaskUtils.getDateTimeFormatterForDataset(timeSpec, zone);
    Assert.assertEquals(dateTimeFormatter.print(dateTimeInMS), "20170112");
    zone = DateTimeZone.forID("America/Los_Angeles");
    long dateTimeInMS1 = new DateTime(2017, 01, 12, 05, 30, zone).getMillis();
    // DAYS bucket
    timeGranularity = new TimeGranularity(1, TimeUnit.DAYS);
    timeSpec = new TimeSpec(columnName, timeGranularity, timeFormat);
    dateTimeFormatter = DataCompletenessTaskUtils.getDateTimeFormatterForDataset(timeSpec, zone);
    Assert.assertEquals(dateTimeFormatter.print(dateTimeInMS1), "20170112");
    // HOURS bucket
    zone = DateTimeZone.UTC;
    dateTimeInMS = new DateTime(2017, 01, 12, 15, 30, zone).getMillis();
    timeGranularity = new TimeGranularity(1, TimeUnit.HOURS);
    timeSpec = new TimeSpec(columnName, timeGranularity, timeFormat);
    dateTimeFormatter = DataCompletenessTaskUtils.getDateTimeFormatterForDataset(timeSpec, zone);
    Assert.assertEquals(dateTimeFormatter.print(dateTimeInMS), "2017011215");
    // MINUTES bucket
    timeGranularity = new TimeGranularity(1, TimeUnit.MINUTES);
    timeSpec = new TimeSpec(columnName, timeGranularity, timeFormat);
    dateTimeFormatter = DataCompletenessTaskUtils.getDateTimeFormatterForDataset(timeSpec, zone);
    Assert.assertEquals(dateTimeFormatter.print(dateTimeInMS), "201701121530");
    // DEFAULT bucket
    timeGranularity = new TimeGranularity(1, TimeUnit.MILLISECONDS);
    timeSpec = new TimeSpec(columnName, timeGranularity, timeFormat);
    dateTimeFormatter = DataCompletenessTaskUtils.getDateTimeFormatterForDataset(timeSpec, zone);
    Assert.assertEquals(dateTimeFormatter.print(dateTimeInMS), "2017011215");
}

Example 39 with DateTimeFormatter

use of org.joda.time.format.DateTimeFormatter in project pinot by linkedin.

the class TestDetectionJobSchedulerUtils method testGetNewEntriesForDetectionSchedulerHourly.

@Test
public void testGetNewEntriesForDetectionSchedulerHourly() throws Exception {
    DatasetConfigDTO datasetConfig = new DatasetConfigDTO();
    datasetConfig.setTimeColumn("Date");
    datasetConfig.setTimeUnit(TimeUnit.HOURS);
    datasetConfig.setTimeDuration(1);
    DateTimeZone dateTimeZone = DateTimeZone.UTC;
    AnomalyFunctionDTO anomalyFunction = new AnomalyFunctionDTO();
    DateTimeFormatter dateTimeFormatter = DetectionJobSchedulerUtils.getDateTimeFormatterForDataset(datasetConfig, dateTimeZone);
    String currentDateTimeString = "201702140336";
    String currentDateTimeStringRounded = "2017021403";
    DateTime currentDateTime = minuteDateTimeFormatter.parseDateTime(currentDateTimeString);
    DateTime currentDateTimeRounded = dateTimeFormatter.parseDateTime(currentDateTimeStringRounded);
    DetectionStatusDTO lastEntryForFunction = null;
    // null last entry
    Map<String, Long> newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 1);
    Assert.assertEquals(newEntries.get(currentDateTimeStringRounded), new Long(currentDateTimeRounded.getMillis()));
    // last entry same as current time
    lastEntryForFunction = new DetectionStatusDTO();
    lastEntryForFunction.setDateToCheckInSDF(currentDateTimeStringRounded);
    lastEntryForFunction.setDateToCheckInMS(currentDateTimeRounded.getMillis());
    newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 0);
    // last entry 1 hour before current time
    String lastEntryDateTimeString = "2017021402";
    DateTime lastEntryDateTime = dateTimeFormatter.parseDateTime(lastEntryDateTimeString);
    lastEntryForFunction = new DetectionStatusDTO();
    lastEntryForFunction.setDateToCheckInSDF(lastEntryDateTimeString);
    lastEntryForFunction.setDateToCheckInMS(lastEntryDateTime.getMillis());
    newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 1);
    Assert.assertEquals(newEntries.get(currentDateTimeStringRounded), new Long(currentDateTimeRounded.getMillis()));
    // last entry 3 hours before current time
    lastEntryDateTimeString = "2017021400";
    lastEntryDateTime = dateTimeFormatter.parseDateTime(lastEntryDateTimeString);
    lastEntryForFunction = new DetectionStatusDTO();
    lastEntryForFunction.setDateToCheckInSDF(lastEntryDateTimeString);
    lastEntryForFunction.setDateToCheckInMS(lastEntryDateTime.getMillis());
    newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 3);
    Assert.assertNotNull(newEntries.get("2017021401"));
    Assert.assertNotNull(newEntries.get("2017021402"));
    Assert.assertNotNull(newEntries.get("2017021403"));
    Assert.assertEquals(newEntries.get(currentDateTimeStringRounded), new Long(currentDateTimeRounded.getMillis()));
}

Example 40 with DateTimeFormatter

use of org.joda.time.format.DateTimeFormatter in project druid by druid-io.

the class GranularityPathSpec method addInputPaths.

@Override
public Job addInputPaths(HadoopDruidIndexerConfig config, Job job) throws IOException {
    final Set<Interval> intervals = Sets.newTreeSet(Comparators.intervalsByStartThenEnd());
    for (Interval inputInterval : config.getInputIntervals()) {
        for (Interval interval : dataGranularity.getIterable(inputInterval)) {
            intervals.add(trim(inputInterval, interval));
        }
    }
    Path betaInput = new Path(inputPath);
    FileSystem fs = betaInput.getFileSystem(job.getConfiguration());
    Set<String> paths = Sets.newTreeSet();
    Pattern fileMatcher = Pattern.compile(filePattern);
    DateTimeFormatter customFormatter = null;
    if (pathFormat != null) {
        customFormatter = DateTimeFormat.forPattern(pathFormat);
    }
    for (Interval interval : intervals) {
        DateTime t = interval.getStart();
        String intervalPath = null;
        if (customFormatter != null) {
            intervalPath = customFormatter.print(t);
        } else {
            intervalPath = dataGranularity.toPath(t);
        }
        Path granularPath = new Path(betaInput, intervalPath);
        log.info("Checking path[%s]", granularPath);
        for (FileStatus status : FSSpideringIterator.spiderIterable(fs, granularPath)) {
            final Path filePath = status.getPath();
            if (fileMatcher.matcher(filePath.toString()).matches()) {
                paths.add(filePath.toString());
            }
        }
    }
    log.info("Appending path %s", paths);
    StaticPathSpec.addToMultipleInputs(config, job, paths, inputFormat);
    return job;
}