Examples with ResultSetGroup com.linkedin.pinot.client.ResultSetGroup used on opensource projects

Example 6 with ResultSetGroup

use of com.linkedin.pinot.client.ResultSetGroup in project pinot by linkedin.

the class CollectionMaxDataTimeCacheLoader method load.

@Override
public Long load(String collection) throws Exception {
    LOGGER.info("Loading maxDataTime cache {}", collection);
    long maxTime = 0;
    try {
        DatasetConfigDTO datasetConfig = datasetConfigDAO.findByDataset(collection);
        // By default, query only offline, unless dataset has been marked as realtime
        String tableName = ThirdEyeUtils.computeTableName(collection);
        TimeSpec timeSpec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
        long prevMaxDataTime = getPrevMaxDataTime(collection, timeSpec);
        String maxTimePql = String.format(COLLECTION_MAX_TIME_QUERY_TEMPLATE, timeSpec.getColumnName(), tableName, timeSpec.getColumnName(), prevMaxDataTime);
        PinotQuery maxTimePinotQuery = new PinotQuery(maxTimePql, tableName);
        resultSetGroupCache.refresh(maxTimePinotQuery);
        ResultSetGroup resultSetGroup = resultSetGroupCache.get(maxTimePinotQuery);
        if (resultSetGroup.getResultSetCount() == 0 || resultSetGroup.getResultSet(0).getRowCount() == 0) {
            LOGGER.info("resultSetGroup is Empty for collection {} is {}", tableName, resultSetGroup);
            this.collectionToPrevMaxDataTimeMap.remove(collection);
        } else {
            long endTime = new Double(resultSetGroup.getResultSet(0).getDouble(0)).longValue();
            this.collectionToPrevMaxDataTimeMap.put(collection, endTime);
            // endTime + 1 to make sure we cover the time range of that time value.
            String timeFormat = timeSpec.getFormat();
            if (StringUtils.isBlank(timeFormat) || TimeSpec.SINCE_EPOCH_FORMAT.equals(timeFormat)) {
                maxTime = timeSpec.getDataGranularity().toMillis(endTime + 1) - 1;
            } else {
                DateTimeFormatter inputDataDateTimeFormatter = DateTimeFormat.forPattern(timeFormat).withZone(Utils.getDataTimeZone(collection));
                maxTime = DateTime.parse(String.valueOf(endTime), inputDataDateTimeFormatter).getMillis();
            }
        }
    } catch (Exception e) {
        LOGGER.warn("Exception getting maxTime from collection: {}", collection, e);
        this.collectionToPrevMaxDataTimeMap.remove(collection);
    }
    if (maxTime <= 0) {
        maxTime = System.currentTimeMillis();
    }
    return maxTime;
}

Example 7 with ResultSetGroup

use of com.linkedin.pinot.client.ResultSetGroup in project pinot by linkedin.

the class DataCompletenessTaskUtils method getBucketNameToCountStarMap.

private static Map<String, Long> getBucketNameToCountStarMap(String dataset, TimeSpec timeSpec, ListMultimap<String, Long> bucketNameToTimeValues) {
    Map<String, Long> bucketNameToCountStar = new HashMap<>();
    // generate request
    StringBuilder sb = new StringBuilder();
    String delimiter = "";
    for (Long timeValue : bucketNameToTimeValues.values()) {
        sb.append(delimiter);
        delimiter = " OR ";
        sb.append(String.format("%s='%s'", timeSpec.getColumnName(), timeValue));
    }
    long top = bucketNameToTimeValues.values().size();
    String pql = String.format("select count(*) from %s where %s group by %s top %s", dataset, sb.toString(), timeSpec.getColumnName(), top);
    Map<Long, Long> timeValueToCount = new HashMap<>();
    try {
        ResultSetGroup resultSetGroup = CACHE_REGISTRY.getResultSetGroupCache().get(new PinotQuery(pql, dataset));
        if (resultSetGroup == null || resultSetGroup.getResultSetCount() <= 0) {
            return bucketNameToCountStar;
        }
        ResultSet resultSet = resultSetGroup.getResultSet(0);
        for (int i = 0; i < resultSet.getRowCount(); i++) {
            Long timeValue = Long.valueOf(resultSet.getGroupKeyString(i, 0));
            Long count = resultSet.getLong(i, 0);
            timeValueToCount.put(timeValue, count);
        }
    } catch (ExecutionException e) {
        LOG.error("Exception in getting count *. PQL:{}", pql, e);
    }
    // parse response to get counts
    for (String bucketName : bucketNameToTimeValues.keySet()) {
        List<Long> timeValues = bucketNameToTimeValues.get(bucketName);
        Long sumOfCountForBucket = 0L;
        for (Long timeValue : timeValues) {
            sumOfCountForBucket = sumOfCountForBucket + timeValueToCount.getOrDefault(timeValue, 0L);
        }
        bucketNameToCountStar.put(bucketName, sumOfCountForBucket);
    }
    return bucketNameToCountStar;
}