Examples with SegmentMetadata com.linkedin.pinot.common.segment.SegmentMetadata used on opensource projects

Example 11 with SegmentMetadata

use of com.linkedin.pinot.common.segment.SegmentMetadata in project pinot by linkedin.

the class ColumnarSegmentMetadataLoader method load.

@Override
public SegmentMetadata load(File segmentDir) throws Exception {
    final SegmentMetadata segmentMetadata = new SegmentMetadataImpl(segmentDir);
    LOGGER.info("Loaded segment metadata for segment : " + segmentMetadata.getName());
    return segmentMetadata;
}

Example 12 with SegmentMetadata

use of com.linkedin.pinot.common.segment.SegmentMetadata in project pinot by linkedin.

the class RetentionManagerTest method testRetentionWithMillsTimeUnit.

/**
   * Test with millseconds time unit.
   * @throws JSONException
   * @throws UnsupportedEncodingException
   * @throws IOException
   * @throws InterruptedException
   */
@Test
public void testRetentionWithMillsTimeUnit() throws JSONException, UnsupportedEncodingException, IOException, InterruptedException {
    _retentionManager = new RetentionManager(_pinotHelixResourceManager, 5);
    _retentionManager.start();
    long theDayAfterTomorrowSinceEpoch = System.currentTimeMillis() / 1000 / 60 / 60 / 24 + 2;
    long millsSinceEpochTimeStamp = theDayAfterTomorrowSinceEpoch * 24 * 60 * 60 * 1000;
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl("1343001600000", "1343001600000", TimeUnit.MILLISECONDS.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl(millsSinceEpochTimeStamp + "", millsSinceEpochTimeStamp + "", TimeUnit.MILLISECONDS.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    validate(20, _offlineTableName, 10, true);
    cleanupSegments(_offlineTableName);
}

Example 13 with SegmentMetadata

use of com.linkedin.pinot.common.segment.SegmentMetadata in project pinot by linkedin.

the class RetentionManagerTest method testRetentionWithSecondsTimeUnit.

/**
   * @throws JSONException
   * @throws UnsupportedEncodingException
   * @throws IOException
   * @throws InterruptedException
   */
@Test
public void testRetentionWithSecondsTimeUnit() throws JSONException, UnsupportedEncodingException, IOException, InterruptedException {
    _retentionManager = new RetentionManager(_pinotHelixResourceManager, 5);
    _retentionManager.start();
    long theDayAfterTomorrowSinceEpoch = System.currentTimeMillis() / 1000 / 60 / 60 / 24 + 2;
    long secondsSinceEpochTimeStamp = theDayAfterTomorrowSinceEpoch * 24 * 60 * 60;
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl("1343001600", "1343001600", TimeUnit.SECONDS.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl(secondsSinceEpochTimeStamp + "", secondsSinceEpochTimeStamp + "", TimeUnit.SECONDS.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    validate(20, _offlineTableName, 10, true);
    cleanupSegments(_offlineTableName);
}

Example 14 with SegmentMetadata

use of com.linkedin.pinot.common.segment.SegmentMetadata in project pinot by linkedin.

the class RetentionManagerTest method testRetentionWithMinutesTimeUnit.

/**
   * @throws JSONException
   * @throws UnsupportedEncodingException
   * @throws IOException
   * @throws InterruptedException
   */
@Test
public void testRetentionWithMinutesTimeUnit() throws JSONException, UnsupportedEncodingException, IOException, InterruptedException {
    _retentionManager = new RetentionManager(_pinotHelixResourceManager, 5);
    _retentionManager.start();
    long theDayAfterTomorrowSinceEpoch = System.currentTimeMillis() / 1000 / 60 / 60 / 24 + 2;
    long minutesSinceEpochTimeStamp = theDayAfterTomorrowSinceEpoch * 24 * 60;
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl("22383360", "22383360", TimeUnit.MINUTES.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl(minutesSinceEpochTimeStamp + "", minutesSinceEpochTimeStamp + "", TimeUnit.MINUTES.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    validate(20, _offlineTableName, 10, true);
    cleanupSegments(_offlineTableName);
}

Example 15 with SegmentMetadata

use of com.linkedin.pinot.common.segment.SegmentMetadata in project pinot by linkedin.

the class RetentionManagerTest method getTimeSegmentMetadataImpl.

private SegmentMetadata getTimeSegmentMetadataImpl(final String startTime, final String endTime, final String timeUnit) {
    if (startTime == null || endTime == null || timeUnit == null) {
        long startTimeValue = System.currentTimeMillis();
        return getTimeSegmentMetadataImpl(startTimeValue + "", startTimeValue + "", TimeUnit.MILLISECONDS.toString());
    }
    final long creationTime = System.currentTimeMillis();
    final String segmentName = _testTableName + creationTime;
    SegmentMetadata segmentMetadata = new SegmentMetadata() {

        TimeUnit segmentTimeUnit = TimeUtils.timeUnitFromString(timeUnit);

        Duration _timeGranularity = new Duration(segmentTimeUnit.toMillis(1));

        Interval _timeInterval = new Interval(segmentTimeUnit.toMillis(Long.parseLong(startTime)), segmentTimeUnit.toMillis(Long.parseLong(endTime)));

        @Override
        public Map<String, String> toMap() {
            final Map<String, String> ret = new HashMap<String, String>();
            ret.put(V1Constants.MetadataKeys.Segment.TABLE_NAME, getTableName());
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_TOTAL_DOCS, String.valueOf(getTotalDocs()));
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_VERSION, getVersion());
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_NAME, getName());
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_CRC, getCrc());
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_CREATION_TIME, getIndexCreationTime() + "");
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_START_TIME, startTime);
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_END_TIME, endTime);
            ret.put(V1Constants.MetadataKeys.Segment.TIME_UNIT, timeUnit);
            return ret;
        }

        @Override
        public String getVersion() {
            return SegmentVersion.v1.toString();
        }

        @Override
        public int getTotalDocs() {
            return 0;
        }

        @Override
        public int getTotalRawDocs() {
            return 0;
        }

        @Override
        public Interval getTimeInterval() {
            return _timeInterval;
        }

        @Override
        public Duration getTimeGranularity() {
            return _timeGranularity;
        }

        @Override
        public String getShardingKey() {
            return null;
        }

        @Override
        public Schema getSchema() {
            return null;
        }

        @Override
        public String getTableName() {
            return _testTableName;
        }

        @Override
        public String getName() {
            return segmentName;
        }

        @Override
        public String getIndexType() {
            return "offline";
        }

        @Override
        public String getTimeColumn() {
            return null;
        }

        @Override
        public long getStartTime() {
            return Long.valueOf(startTime);
        }

        @Override
        public long getEndTime() {
            return Long.valueOf(endTime);
        }

        @Override
        public TimeUnit getTimeUnit() {
            return segmentTimeUnit;
        }

        @Override
        public String getIndexDir() {
            return null;
        }

        @Override
        public long getIndexCreationTime() {
            return creationTime;
        }

        @Override
        public String getCrc() {
            return creationTime + "";
        }

        @Override
        public long getPushTime() {
            return Long.MIN_VALUE;
        }

        @Override
        public long getRefreshTime() {
            return Long.MIN_VALUE;
        }

        @Override
        public boolean hasDictionary(String columnName) {
            return false;
        }

        @Override
        public boolean hasStarTree() {
            return false;
        }

        @Override
        public StarTreeMetadata getStarTreeMetadata() {
            return null;
        }

        @Override
        public boolean close() {
            return false;
        }

        @Override
        public String getForwardIndexFileName(String column, String segmentVersion) {
            throw new UnsupportedOperationException("getForwardIndexFileName not supported in " + this.getClass());
        }

        @Override
        public String getDictionaryFileName(String column, String segmentVersion) {
            throw new UnsupportedOperationException("getDictionaryFileName not supported in " + this.getClass());
        }

        @Override
        public String getBitmapInvertedIndexFileName(String column, String segmentVersion) {
            throw new UnsupportedOperationException("getBitmapInvertedIndexFileName not supported in " + this.getClass());
        }

        @Nullable
        @Override
        public String getCreatorName() {
            return null;
        }

        @Override
        public char getPaddingCharacter() {
            return V1Constants.Str.DEFAULT_STRING_PAD_CHAR;
        }

        @Override
        public int getHllLog2m() {
            return HllConstants.DEFAULT_LOG2M;
        }

        @Nullable
        @Override
        public String getDerivedColumn(String column, MetricFieldSpec.DerivedMetricType derivedMetricType) {
            return null;
        }
    };
    return segmentMetadata;
}