Examples with TimeUnit java.util.concurrent.TimeUnit used on opensource projects

Example 31 with TimeUnit

use of java.util.concurrent.TimeUnit in project pinot by linkedin.

the class TableRetentionValidator method run.

public void run() throws Exception {
    // Get all resources in cluster
    List<String> resourcesInCluster = _helixAdmin.getResourcesInCluster(_clusterName);
    for (String tableName : resourcesInCluster) {
        // Skip non-table resources
        if (!tableName.endsWith("_OFFLINE") && !tableName.endsWith("_REALTIME")) {
            continue;
        }
        // Skip tables that do not match the defined name pattern
        if (_tableNamePattern != null && !tableName.matches(_tableNamePattern)) {
            continue;
        }
        // Get the retention config
        SegmentsValidationAndRetentionConfig retentionConfig = getTableConfig(tableName).getValidationConfig();
        if (retentionConfig == null) {
            LOGGER.error("Table: {}, \"segmentsConfig\" field is missing in table config", tableName);
            continue;
        }
        String segmentPushType = retentionConfig.getSegmentPushType();
        if (segmentPushType == null) {
            LOGGER.error("Table: {}, null push type", tableName);
            continue;
        } else if (segmentPushType.equalsIgnoreCase("REFRESH")) {
            continue;
        } else if (!segmentPushType.equalsIgnoreCase("APPEND")) {
            LOGGER.error("Table: {}, invalid push type: {}", tableName, segmentPushType);
            continue;
        }
        // APPEND use case
        // Get time unit
        String timeUnitString = retentionConfig.getRetentionTimeUnit();
        TimeUnit timeUnit;
        try {
            timeUnit = TimeUnit.valueOf(timeUnitString.toUpperCase());
        } catch (Exception e) {
            LOGGER.error("Table: {}, invalid time unit: {}", tableName, timeUnitString);
            continue;
        }
        // Get time duration in days
        String timeValueString = retentionConfig.getRetentionTimeValue();
        long durationInDays;
        try {
            durationInDays = timeUnit.toDays(Long.valueOf(timeValueString));
        } catch (Exception e) {
            LOGGER.error("Table: {}, invalid time value: {}", tableName, timeValueString);
            continue;
        }
        if (durationInDays <= 0) {
            LOGGER.error("Table: {}, invalid retention duration in days: {}", tableName, durationInDays);
            continue;
        }
        if (durationInDays > _durationInDaysThreshold) {
            LOGGER.warn("Table: {}, retention duration in days is too large: {}", tableName, durationInDays);
        }
        // Skip segments metadata check for realtime tables
        if (tableName.endsWith("REALTIME")) {
            continue;
        }
        // Check segments metadata (only for offline tables)
        List<String> segmentNames = getSegmentNames(tableName);
        if (segmentNames == null || segmentNames.isEmpty()) {
            LOGGER.warn("Table: {}, no segment metadata in property store", tableName);
            continue;
        }
        List<String> errorMessages = new ArrayList<>();
        for (String segmentName : segmentNames) {
            OfflineSegmentZKMetadata offlineSegmentMetadata = getOfflineSegmentMetadata(tableName, segmentName);
            TimeUnit segmentTimeUnit = offlineSegmentMetadata.getTimeUnit();
            if (segmentTimeUnit == null) {
                errorMessages.add("Segment: " + segmentName + " has null time unit");
                continue;
            }
            long startTimeInMillis = segmentTimeUnit.toMillis(offlineSegmentMetadata.getStartTime());
            if (!TimeUtils.timeValueInValidRange(startTimeInMillis)) {
                errorMessages.add("Segment: " + segmentName + " has invalid start time in millis: " + startTimeInMillis);
            }
            long endTimeInMillis = segmentTimeUnit.toMillis(offlineSegmentMetadata.getEndTime());
            if (!TimeUtils.timeValueInValidRange(endTimeInMillis)) {
                errorMessages.add("Segment: " + segmentName + " has invalid end time in millis: " + endTimeInMillis);
            }
        }
        if (!errorMessages.isEmpty()) {
            LOGGER.error("Table: {}, invalid segments: {}", tableName, errorMessages);
        }
    }
}

Example 32 with TimeUnit

use of java.util.concurrent.TimeUnit in project pinot by linkedin.

the class TimeRetentionStrategy method isPurgeable.

@Override
public boolean isPurgeable(SegmentZKMetadata segmentZKMetadata) {
    if (_retentionDuration == null || _retentionDuration.getMillis() <= 0) {
        return false;
    }
    try {
        TimeUnit segmentTimeUnit = segmentZKMetadata.getTimeUnit();
        if (segmentTimeUnit == null) {
            return false;
        }
        long endsMillis = segmentTimeUnit.toMillis(segmentZKMetadata.getEndTime());
        // Check that the date in the segment is between 1971 and 2071, as a sanity check for misconfigured time units.
        if (!TimeUtils.timeValueInValidRange(endsMillis)) {
            LOGGER.warn("Skipping purge check for segment {}, timestamp {} {} fails sanity check.", segmentZKMetadata.getSegmentName(), segmentZKMetadata.getEndTime(), segmentZKMetadata.getTimeUnit());
            return false;
        }
        Duration segmentTimeUntilNow = new Duration(endsMillis, System.currentTimeMillis());
        if (_retentionDuration.isShorterThan(segmentTimeUntilNow)) {
            return true;
        }
    } catch (Exception e) {
        LOGGER.warn("Caught exception while checking if a segment is purgeable", e);
        return false;
    }
    return false;
}

Example 33 with TimeUnit

use of java.util.concurrent.TimeUnit in project pinot by linkedin.

the class RetentionManagerTest method getTimeSegmentMetadataImpl.

private SegmentMetadata getTimeSegmentMetadataImpl(final String startTime, final String endTime, final String timeUnit) {
    if (startTime == null || endTime == null || timeUnit == null) {
        long startTimeValue = System.currentTimeMillis();
        return getTimeSegmentMetadataImpl(startTimeValue + "", startTimeValue + "", TimeUnit.MILLISECONDS.toString());
    }
    final long creationTime = System.currentTimeMillis();
    final String segmentName = _testTableName + creationTime;
    SegmentMetadata segmentMetadata = new SegmentMetadata() {

        TimeUnit segmentTimeUnit = TimeUtils.timeUnitFromString(timeUnit);

        Duration _timeGranularity = new Duration(segmentTimeUnit.toMillis(1));

        Interval _timeInterval = new Interval(segmentTimeUnit.toMillis(Long.parseLong(startTime)), segmentTimeUnit.toMillis(Long.parseLong(endTime)));

        @Override
        public Map<String, String> toMap() {
            final Map<String, String> ret = new HashMap<String, String>();
            ret.put(V1Constants.MetadataKeys.Segment.TABLE_NAME, getTableName());
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_TOTAL_DOCS, String.valueOf(getTotalDocs()));
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_VERSION, getVersion());
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_NAME, getName());
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_CRC, getCrc());
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_CREATION_TIME, getIndexCreationTime() + "");
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_START_TIME, startTime);
            ret.put(V1Constants.MetadataKeys.Segment.SEGMENT_END_TIME, endTime);
            ret.put(V1Constants.MetadataKeys.Segment.TIME_UNIT, timeUnit);
            return ret;
        }

        @Override
        public String getVersion() {
            return SegmentVersion.v1.toString();
        }

        @Override
        public int getTotalDocs() {
            return 0;
        }

        @Override
        public int getTotalRawDocs() {
            return 0;
        }

        @Override
        public Interval getTimeInterval() {
            return _timeInterval;
        }

        @Override
        public Duration getTimeGranularity() {
            return _timeGranularity;
        }

        @Override
        public String getShardingKey() {
            return null;
        }

        @Override
        public Schema getSchema() {
            return null;
        }

        @Override
        public String getTableName() {
            return _testTableName;
        }

        @Override
        public String getName() {
            return segmentName;
        }

        @Override
        public String getIndexType() {
            return "offline";
        }

        @Override
        public String getTimeColumn() {
            return null;
        }

        @Override
        public long getStartTime() {
            return Long.valueOf(startTime);
        }

        @Override
        public long getEndTime() {
            return Long.valueOf(endTime);
        }

        @Override
        public TimeUnit getTimeUnit() {
            return segmentTimeUnit;
        }

        @Override
        public String getIndexDir() {
            return null;
        }

        @Override
        public long getIndexCreationTime() {
            return creationTime;
        }

        @Override
        public String getCrc() {
            return creationTime + "";
        }

        @Override
        public long getPushTime() {
            return Long.MIN_VALUE;
        }

        @Override
        public long getRefreshTime() {
            return Long.MIN_VALUE;
        }

        @Override
        public boolean hasDictionary(String columnName) {
            return false;
        }

        @Override
        public boolean hasStarTree() {
            return false;
        }

        @Override
        public StarTreeMetadata getStarTreeMetadata() {
            return null;
        }

        @Override
        public boolean close() {
            return false;
        }

        @Override
        public String getForwardIndexFileName(String column, String segmentVersion) {
            throw new UnsupportedOperationException("getForwardIndexFileName not supported in " + this.getClass());
        }

        @Override
        public String getDictionaryFileName(String column, String segmentVersion) {
            throw new UnsupportedOperationException("getDictionaryFileName not supported in " + this.getClass());
        }

        @Override
        public String getBitmapInvertedIndexFileName(String column, String segmentVersion) {
            throw new UnsupportedOperationException("getBitmapInvertedIndexFileName not supported in " + this.getClass());
        }

        @Nullable
        @Override
        public String getCreatorName() {
            return null;
        }

        @Override
        public char getPaddingCharacter() {
            return V1Constants.Str.DEFAULT_STRING_PAD_CHAR;
        }

        @Override
        public int getHllLog2m() {
            return HllConstants.DEFAULT_LOG2M;
        }

        @Nullable
        @Override
        public String getDerivedColumn(String column, MetricFieldSpec.DerivedMetricType derivedMetricType) {
            return null;
        }
    };
    return segmentMetadata;
}

Example 34 with TimeUnit

use of java.util.concurrent.TimeUnit in project pinot by linkedin.

the class TimeConversionTransform method transform.

@Override
public <T> T transform(int length, BlockValSet... input) {
    Preconditions.checkArgument(input.length == 3, TRANSFORM_NAME + " expects three arguments");
    long[] inputTime = input[0].getLongValuesSV();
    String[] inputTimeUnits = input[1].getStringValuesSV();
    String[] outputTimeUnits = input[2].getStringValuesSV();
    Preconditions.checkState(inputTime.length >= length && inputTimeUnits.length >= length, outputTimeUnits.length >= length);
    TimeUnit inputTimeUnit = TimeUnit.valueOf(inputTimeUnits[0]);
    TimeUnit outputTimeUnit = TimeUnit.valueOf(outputTimeUnits[0]);
    if (_output == null || _output.length < length) {
        _output = new long[Math.max(length, DocIdSetPlanNode.MAX_DOC_PER_CALL)];
    }
    for (int i = 0; i < length; i++) {
        _output[i] = outputTimeUnit.convert(inputTime[i], inputTimeUnit);
    }
    return (T) _output;
}

Example 35 with TimeUnit

use of java.util.concurrent.TimeUnit in project pinot by linkedin.

the class FileBasedSentineTest method setup.

@BeforeClass
public void setup() throws Exception {
    url = new URL("http://localhost:" + FileBasedServerBrokerStarters.BROKER_CLIENT_PORT + "/query");
    // lets generate data
    final String[] columns = { "dimention1", "dimention2", "dimention3", "dimention4", "metric1", "daysSinceEpoch" };
    final Map<String, DataType> dataTypes = new HashMap<String, FieldSpec.DataType>();
    final Map<String, FieldType> fieldTypes = new HashMap<String, FieldType>();
    final Map<String, TimeUnit> timeUnits = new HashMap<String, TimeUnit>();
    final Map<String, Integer> cardinality = new HashMap<String, Integer>();
    // Crate empty range map as the signature of DataGeneratorSpec has changed, and this test does not
    // use metric/time as fieldType.
    final Map<String, IntRange> range = new HashMap<String, IntRange>();
    for (final String col : columns) {
        if (col.equals("dimention1")) {
            dataTypes.put(col, DataType.STRING);
            cardinality.put(col, 1000);
        } else {
            dataTypes.put(col, DataType.INT);
            cardinality.put(col, 1000);
        }
        fieldTypes.put(col, FieldType.DIMENSION);
    }
    if (avroDataDir.exists()) {
        FileUtils.deleteDirectory(avroDataDir);
    }
    final DataGeneratorSpec spec = new DataGeneratorSpec(Arrays.asList(columns), cardinality, range, dataTypes, fieldTypes, timeUnits, FileFormat.AVRO, avroDataDir.getAbsolutePath(), true);
    generator = new DataGenerator();
    generator.init(spec);
    generator.generate(100000L, 2);
    // lets make segments now
    final File bootstrapDir = new File(FileBasedServerBrokerStarters.SERVER_BOOTSTRAP_DIR);
    if (bootstrapDir.exists()) {
        FileUtils.deleteDirectory(bootstrapDir);
    }
    bootstrapDir.mkdir();
    int counter = 0;
    for (final File avro : avroDataDir.listFiles()) {
        for (final String table : FileBasedServerBrokerStarters.TABLE_NAMES) {
            final SegmentGeneratorConfig genConfig = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(avro, new File(bootstrapDir, "segment-" + counter), "daysSinceEpoch", TimeUnit.DAYS, table);
            final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
            driver.init(genConfig);
            driver.build();
            counter++;
        }
    }
    // lets start the server and the broker now
    starter = new FileBasedServerBrokerStarters();
    starter.startAll();
    // pick some values from here if you need to use it for running filter queries
    final JSONObject selectionRequestResponse = postQuery("select * from 'table1' limit 100", "http://localhost:" + FileBasedServerBrokerStarters.BROKER_CLIENT_PORT);
//    System.out.println(selectionRequestResponse.toString(1));
}