Examples with Interval org.joda.time.Interval used on opensource projects

Example 86 with Interval

use of org.joda.time.Interval in project pinot by linkedin.

the class ValidationManager method validateOfflineSegmentPush.

// For offline segment pushes, validate that there are no missing segments, and update metrics
private void validateOfflineSegmentPush(ZkHelixPropertyStore<ZNRecord> propertyStore, String tableName, List<SegmentMetadata> segmentMetadataList) {
    List<OfflineSegmentZKMetadata> offlineSegmentZKMetadatas = ZKMetadataProvider.getOfflineSegmentZKMetadataListForTable(propertyStore, tableName);
    for (OfflineSegmentZKMetadata offlineSegmentZKMetadata : offlineSegmentZKMetadatas) {
        SegmentMetadata segmentMetadata = new SegmentMetadataImpl(offlineSegmentZKMetadata);
        segmentMetadataList.add(segmentMetadata);
    }
    // Calculate missing segments only for offline tables
    int missingSegmentCount = 0;
    // Compute the missing segments if there are at least two
    if (2 < segmentMetadataList.size()) {
        List<Interval> segmentIntervals = new ArrayList<Interval>();
        for (SegmentMetadata segmentMetadata : segmentMetadataList) {
            Interval timeInterval = segmentMetadata.getTimeInterval();
            if (timeInterval != null && TimeUtils.timeValueInValidRange(timeInterval.getStartMillis()) && TimeUtils.timeValueInValidRange(timeInterval.getEndMillis())) {
                segmentIntervals.add(timeInterval);
            }
        }
        List<Interval> missingIntervals = computeMissingIntervals(segmentIntervals, segmentMetadataList.get(0).getTimeGranularity());
        missingSegmentCount = missingIntervals.size();
        for (Interval missingInterval : missingIntervals) {
            LOGGER.warn("Missing data in table {} for time interval {}", tableName, missingInterval);
        }
    }
    // Update the gauge that contains the number of missing segments
    _validationMetrics.updateMissingSegmentsGauge(tableName, missingSegmentCount);
    // Compute the max segment end time and max segment push time
    long maxSegmentEndTime = Long.MIN_VALUE;
    long maxSegmentPushTime = Long.MIN_VALUE;
    for (SegmentMetadata segmentMetadata : segmentMetadataList) {
        Interval segmentInterval = segmentMetadata.getTimeInterval();
        if (segmentInterval != null && maxSegmentEndTime < segmentInterval.getEndMillis()) {
            maxSegmentEndTime = segmentInterval.getEndMillis();
        }
        long segmentPushTime = segmentMetadata.getPushTime();
        long segmentRefreshTime = segmentMetadata.getRefreshTime();
        long segmentUpdateTime = Math.max(segmentPushTime, segmentRefreshTime);
        if (maxSegmentPushTime < segmentUpdateTime) {
            maxSegmentPushTime = segmentUpdateTime;
        }
    }
    // Update the gauges that contain the delay between the current time and last segment end time
    _validationMetrics.updateOfflineSegmentDelayGauge(tableName, maxSegmentEndTime);
    _validationMetrics.updateLastPushTimeGauge(tableName, maxSegmentPushTime);
    // Update the gauge to contain the total document count in the segments
    _validationMetrics.updateTotalDocumentsGauge(tableName, computeOfflineTotalDocumentInSegments(segmentMetadataList));
    // Update the gauge to contain the total number of segments for this table
    _validationMetrics.updateSegmentCountGauge(tableName, segmentMetadataList.size());
}

Example 87 with Interval

use of org.joda.time.Interval in project pinot by linkedin.

the class ValidationManagerTest method testComputeMissingIntervals.

@Test
public void testComputeMissingIntervals() {
    Interval jan1st = new Interval(new DateTime(2015, 1, 1, 0, 0, 0), new DateTime(2015, 1, 1, 23, 59, 59));
    Interval jan2nd = new Interval(new DateTime(2015, 1, 2, 0, 0, 0), new DateTime(2015, 1, 2, 23, 59, 59));
    Interval jan3rd = new Interval(new DateTime(2015, 1, 3, 0, 0, 0), new DateTime(2015, 1, 3, 23, 59, 59));
    Interval jan4th = new Interval(new DateTime(2015, 1, 4, 0, 0, 0), new DateTime(2015, 1, 4, 23, 59, 59));
    Interval jan5th = new Interval(new DateTime(2015, 1, 5, 0, 0, 0), new DateTime(2015, 1, 5, 23, 59, 59));
    ArrayList<Interval> jan1st2nd3rd = new ArrayList<Interval>();
    jan1st2nd3rd.add(jan1st);
    jan1st2nd3rd.add(jan2nd);
    jan1st2nd3rd.add(jan3rd);
    List<Interval> missingIntervalsForJan1st2nd3rd = ValidationManager.computeMissingIntervals(jan1st2nd3rd, Duration.standardDays(1));
    Assert.assertTrue(missingIntervalsForJan1st2nd3rd.isEmpty());
    ArrayList<Interval> jan1st2nd3rd5th = new ArrayList<Interval>(jan1st2nd3rd);
    jan1st2nd3rd5th.add(jan5th);
    List<Interval> missingIntervalsForJan1st2nd3rd5th = ValidationManager.computeMissingIntervals(jan1st2nd3rd5th, Duration.standardDays(1));
    Assert.assertEquals(missingIntervalsForJan1st2nd3rd5th.size(), 1);
    // Should also work if the intervals are in random order
    ArrayList<Interval> jan5th2nd1st = new ArrayList<Interval>();
    jan5th2nd1st.add(jan5th);
    jan5th2nd1st.add(jan2nd);
    jan5th2nd1st.add(jan1st);
    List<Interval> missingIntervalsForJan5th2nd1st = ValidationManager.computeMissingIntervals(jan5th2nd1st, Duration.standardDays(1));
    Assert.assertEquals(missingIntervalsForJan5th2nd1st.size(), 2);
    // Should also work if the intervals are of different sizes
    Interval jan1stAnd2nd = new Interval(new DateTime(2015, 1, 1, 0, 0, 0), new DateTime(2015, 1, 2, 23, 59, 59));
    ArrayList<Interval> jan1st2nd4th5th = new ArrayList<Interval>();
    jan1st2nd4th5th.add(jan1stAnd2nd);
    jan1st2nd4th5th.add(jan4th);
    jan1st2nd4th5th.add(jan5th);
    List<Interval> missingIntervalsForJan1st2nd4th5th = ValidationManager.computeMissingIntervals(jan1st2nd4th5th, Duration.standardDays(1));
    Assert.assertEquals(missingIntervalsForJan1st2nd4th5th.size(), 1);
}

Example 88 with Interval

use of org.joda.time.Interval in project pinot by linkedin.

the class ValidationManagerTest method testExtremeSenario.

@Test
public void testExtremeSenario() {
    List<Interval> intervals = new ArrayList<Interval>();
    intervals.add(new Interval(1, 2));
    intervals.add(new Interval(Integer.MAX_VALUE - 5, Integer.MAX_VALUE));
    intervals.add(new Interval(Integer.MAX_VALUE / 2 - 5, Integer.MAX_VALUE / 2));
    Duration frequency = new Duration(1);
    List<Interval> computeMissingIntervals = ValidationManager.computeMissingIntervals(intervals, frequency);
    Assert.assertEquals(computeMissingIntervals.size(), 22);
}

Example 89 with Interval

use of org.joda.time.Interval in project pinot by linkedin.

the class RetentionManagerTest method getTimeSegmentMetadataImpl.

private SegmentMetadata getTimeSegmentMetadataImpl(final String startTime, final String endTime, final String timeUnit) {
    if (startTime == null || endTime == null || timeUnit == null) {
        long startTimeValue = System.currentTimeMillis();
        return getTimeSegmentMetadataImpl(startTimeValue + "", startTimeValue + "", TimeUnit.MILLISECONDS.toString());
    }
    final long creationTime = System.currentTimeMillis();
    final String segmentName = _testTableName + creationTime;
    SegmentMetadata segmentMetadata = new SegmentMetadata() {

        TimeUnit segmentTimeUnit = TimeUtils.timeUnitFromString(timeUnit);

        Duration _timeGranularity = new Duration(segmentTimeUnit.toMillis(1));

        Interval _timeInterval = new Interval(segmentTimeUnit.toMillis(Long.parseLong(startTime)), segmentTimeUnit.toMillis(Long.parseLong(endTime)));

        @Override
        public Map<String, String> toMap() {
            final Map<String, String> ret = new HashMap<String, String>();
            ret.put(V1Constants.MetadataKeys.Segment.TABLE_NAME, getTableName());
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_TOTAL_DOCS, String.valueOf(getTotalDocs()));
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_VERSION, getVersion());
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_NAME, getName());
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_CRC, getCrc());
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_CREATION_TIME, getIndexCreationTime() + "");
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_START_TIME, startTime);
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_END_TIME, endTime);
            ret.put(V1Constants.MetadataKeys.Segment.TIME_UNIT, timeUnit);
            return ret;
        }

        @Override
        public String getVersion() {
            return SegmentVersion.v1.toString();
        }

        @Override
        public int getTotalDocs() {
            return 0;
        }

        @Override
        public int getTotalRawDocs() {
            return 0;
        }

        @Override
        public Interval getTimeInterval() {
            return _timeInterval;
        }

        @Override
        public Duration getTimeGranularity() {
            return _timeGranularity;
        }

        @Override
        public String getShardingKey() {
            return null;
        }

        @Override
        public Schema getSchema() {
            return null;
        }

        @Override
        public String getTableName() {
            return _testTableName;
        }

        @Override
        public String getName() {
            return segmentName;
        }

        @Override
        public String getIndexType() {
            return "offline";
        }

        @Override
        public String getTimeColumn() {
            return null;
        }

        @Override
        public long getStartTime() {
            return Long.valueOf(startTime);
        }

        @Override
        public long getEndTime() {
            return Long.valueOf(endTime);
        }

        @Override
        public TimeUnit getTimeUnit() {
            return segmentTimeUnit;
        }

        @Override
        public String getIndexDir() {
            return null;
        }

        @Override
        public long getIndexCreationTime() {
            return creationTime;
        }

        @Override
        public String getCrc() {
            return creationTime + "";
        }

        @Override
        public long getPushTime() {
            return Long.MIN_VALUE;
        }

        @Override
        public long getRefreshTime() {
            return Long.MIN_VALUE;
        }

        @Override
        public boolean hasDictionary(String columnName) {
            return false;
        }

        @Override
        public boolean hasStarTree() {
            return false;
        }

        @Override
        public StarTreeMetadata getStarTreeMetadata() {
            return null;
        }

        @Override
        public boolean close() {
            return false;
        }

        @Override
        public String getForwardIndexFileName(String column, String segmentVersion) {
            throw new UnsupportedOperationException("getForwardIndexFileName not supported in " + this.getClass());
        }

        @Override
        public String getDictionaryFileName(String column, String segmentVersion) {
            throw new UnsupportedOperationException("getDictionaryFileName not supported in " + this.getClass());
        }

        @Override
        public String getBitmapInvertedIndexFileName(String column, String segmentVersion) {
            throw new UnsupportedOperationException("getBitmapInvertedIndexFileName not supported in " + this.getClass());
        }

        @Nullable
        @Override
        public String getCreatorName() {
            return null;
        }

        @Override
        public char getPaddingCharacter() {
            return V1Constants.Str.DEFAULT_STRING_PAD_CHAR;
        }

        @Override
        public int getHllLog2m() {
            return HllConstants.DEFAULT_LOG2M;
        }

        @Nullable
        @Override
        public String getDerivedColumn(String column, MetricFieldSpec.DerivedMetricType derivedMetricType) {
            return null;
        }
    };
    return segmentMetadata;
}

Example 90 with Interval

use of org.joda.time.Interval in project pinot by linkedin.

the class RealtimeSegmentImpl method getTimeInterval.

@Override
public Interval getTimeInterval() {
    DateTime start = outgoingGranularitySpec.toDateTime(minTimeVal);
    DateTime end = outgoingGranularitySpec.toDateTime(maxTimeVal);
    return new Interval(start, end);
}