use of com.linkedin.pinot.client.ResultSetGroup in project pinot by linkedin.
the class ResultSetGroupCacheLoader method load.
@Override
public ResultSetGroup load(PinotQuery pinotQuery) throws Exception {
try {
Connection connection = getConnection();
synchronized (connection) {
long start = System.currentTimeMillis();
ResultSetGroup resultSetGroup = connection.execute(pinotQuery.getTableName(), pinotQuery.getPql());
if (LOG.isDebugEnabled()) {
LOG.debug("Query:{} response:{}", pinotQuery.getPql(), format(resultSetGroup));
}
long end = System.currentTimeMillis();
LOG.info("Query:{} took:{} ms", pinotQuery.getPql(), (end - start));
return resultSetGroup;
}
} catch (PinotClientException cause) {
LOG.error("Error when running pql:" + pinotQuery.getPql(), cause);
throw new PinotClientException("Error when running pql:" + pinotQuery.getPql(), cause);
}
}
use of com.linkedin.pinot.client.ResultSetGroup in project pinot by linkedin.
the class ThirdEyeCacheRegistry method initCaches.
private static void initCaches(ThirdEyeConfiguration config) {
ThirdEyeCacheRegistry cacheRegistry = ThirdEyeCacheRegistry.getInstance();
RemovalListener<PinotQuery, ResultSetGroup> listener = new RemovalListener<PinotQuery, ResultSetGroup>() {
@Override
public void onRemoval(RemovalNotification<PinotQuery, ResultSetGroup> notification) {
LOGGER.info("Expired {}", notification.getKey().getPql());
}
};
// ResultSetGroup Cache. The size of this cache is limited by the total number of buckets in all ResultSetGroup.
// We estimate that 1 bucket (including overhead) consumes 1KB and this cache is allowed to use up to 50% of max
// heap space.
long maxBucketNumber = getApproximateMaxBucketNumber(DEFAULT_HEAP_PERCENTAGE_FOR_RESULTSETGROUP_CACHE);
LoadingCache<PinotQuery, ResultSetGroup> resultSetGroupCache = CacheBuilder.newBuilder().removalListener(listener).expireAfterAccess(1, TimeUnit.HOURS).maximumWeight(maxBucketNumber).weigher((pinotQuery, resultSetGroup) -> {
int resultSetCount = resultSetGroup.getResultSetCount();
int weight = 0;
for (int idx = 0; idx < resultSetCount; ++idx) {
com.linkedin.pinot.client.ResultSet resultSet = resultSetGroup.getResultSet(idx);
weight += (resultSet.getColumnCount() * resultSet.getRowCount());
}
return weight;
}).build(new ResultSetGroupCacheLoader(pinotThirdeyeClientConfig));
cacheRegistry.registerResultSetGroupCache(resultSetGroupCache);
LOGGER.info("Max bucket number for ResultSetGroup cache is set to {}", maxBucketNumber);
// CollectionMaxDataTime Cache
LoadingCache<String, Long> collectionMaxDataTimeCache = CacheBuilder.newBuilder().refreshAfterWrite(5, TimeUnit.MINUTES).build(new CollectionMaxDataTimeCacheLoader(resultSetGroupCache, datasetConfigDAO));
cacheRegistry.registerCollectionMaxDataTimeCache(collectionMaxDataTimeCache);
// Query Cache
QueryCache queryCache = new QueryCache(thirdEyeClient, Executors.newFixedThreadPool(10));
cacheRegistry.registerQueryCache(queryCache);
// Dimension Filter cache
LoadingCache<String, String> dimensionFiltersCache = CacheBuilder.newBuilder().build(new DimensionFiltersCacheLoader(cacheRegistry.getQueryCache()));
cacheRegistry.registerDimensionFiltersCache(dimensionFiltersCache);
// Dashboards cache
LoadingCache<String, String> dashboardsCache = CacheBuilder.newBuilder().build(new DashboardsCacheLoader(dashboardConfigDAO));
cacheRegistry.registerDashboardsCache(dashboardsCache);
// Collections cache
CollectionsCache collectionsCache = new CollectionsCache(datasetConfigDAO, config);
cacheRegistry.registerCollectionsCache(collectionsCache);
// DatasetConfig cache
LoadingCache<String, DatasetConfigDTO> datasetConfigCache = CacheBuilder.newBuilder().build(new DatasetConfigCacheLoader(datasetConfigDAO));
cacheRegistry.registerDatasetConfigCache(datasetConfigCache);
// MetricConfig cache
LoadingCache<MetricDataset, MetricConfigDTO> metricConfigCache = CacheBuilder.newBuilder().build(new MetricConfigCacheLoader(metricConfigDAO));
cacheRegistry.registerMetricConfigCache(metricConfigCache);
// DashboardConfigs cache
LoadingCache<String, List<DashboardConfigDTO>> dashboardConfigsCache = CacheBuilder.newBuilder().build(new DashboardConfigCacheLoader(dashboardConfigDAO));
cacheRegistry.registerDashboardConfigsCache(dashboardConfigsCache);
}
use of com.linkedin.pinot.client.ResultSetGroup in project pinot by linkedin.
the class AutoLoadPinotMetricsService method fetchMetricAsADimensionMetrics.
private List<String> fetchMetricAsADimensionMetrics(String dataset, String metricNamesColumn) {
List<String> distinctMetricNames = new ArrayList<>();
ThirdEyeCacheRegistry CACHE_REGISTRY = ThirdEyeCacheRegistry.getInstance();
String sql = String.format("select count(*) from %s group by %s top 10", dataset, metricNamesColumn);
try {
ResultSetGroup result = CACHE_REGISTRY.getResultSetGroupCache().get(new PinotQuery(sql, dataset));
ResultSet resultSet = result.getResultSet(0);
int rowCount = resultSet.getRowCount();
for (int i = 0; i < rowCount; i++) {
String dimensionName = resultSet.getGroupKeyString(i, 0);
distinctMetricNames.add(dimensionName);
}
LOG.info("Distinct Metrics {}", distinctMetricNames);
} catch (Exception e) {
LOG.error("Exception in fetching metrics from pinot", e);
}
return distinctMetricNames;
}
use of com.linkedin.pinot.client.ResultSetGroup in project pinot by linkedin.
the class AnomalyApplicationEndToEndTest method setup.
private void setup() throws Exception {
// Mock query cache
ThirdEyeClient mockThirdeyeClient = Mockito.mock(ThirdEyeClient.class);
Mockito.when(mockThirdeyeClient.execute(Matchers.any(ThirdEyeRequest.class))).thenAnswer(new Answer<ThirdEyeResponse>() {
@Override
public ThirdEyeResponse answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
ThirdEyeRequest request = (ThirdEyeRequest) args[0];
ThirdEyeResponse response = getMockResponse(request);
return response;
}
});
QueryCache mockQueryCache = new QueryCache(mockThirdeyeClient, Executors.newFixedThreadPool(10));
cacheRegistry.registerQueryCache(mockQueryCache);
MetricConfigDTO metricConfig = getTestMetricConfig(collection, metric, 1L);
// create metric config in cache
LoadingCache<MetricDataset, MetricConfigDTO> mockMetricConfigCache = Mockito.mock(LoadingCache.class);
Mockito.when(mockMetricConfigCache.get(new MetricDataset(metric, collection))).thenReturn(metricConfig);
cacheRegistry.registerMetricConfigCache(mockMetricConfigCache);
// create dataset config in cache
LoadingCache<String, DatasetConfigDTO> mockDatasetConfigCache = Mockito.mock(LoadingCache.class);
Mockito.when(mockDatasetConfigCache.get(collection)).thenReturn(getTestDatasetConfig(collection));
cacheRegistry.registerDatasetConfigCache(mockDatasetConfigCache);
ResultSet mockResultSet = Mockito.mock(ResultSet.class);
Mockito.when(mockResultSet.getRowCount()).thenReturn(0);
ResultSetGroup mockResultSetGroup = Mockito.mock(ResultSetGroup.class);
Mockito.when(mockResultSetGroup.getResultSet(0)).thenReturn(mockResultSet);
LoadingCache<PinotQuery, ResultSetGroup> mockResultSetGroupCache = Mockito.mock(LoadingCache.class);
Mockito.when(mockResultSetGroupCache.get(Matchers.any(PinotQuery.class))).thenAnswer(new Answer<ResultSetGroup>() {
@Override
public ResultSetGroup answer(InvocationOnMock invocation) throws Throwable {
return mockResultSetGroup;
}
});
cacheRegistry.registerResultSetGroupCache(mockResultSetGroupCache);
// Application config
thirdeyeAnomalyConfig = new ThirdEyeAnomalyConfiguration();
thirdeyeAnomalyConfig.setId(id);
thirdeyeAnomalyConfig.setDashboardHost(dashboardHost);
MonitorConfiguration monitorConfiguration = new MonitorConfiguration();
monitorConfiguration.setMonitorFrequency(new TimeGranularity(30, TimeUnit.SECONDS));
thirdeyeAnomalyConfig.setMonitorConfiguration(monitorConfiguration);
thirdeyeAnomalyConfig.setRootDir(System.getProperty("dw.rootDir", "NOT_SET(dw.rootDir)"));
// create test anomaly function
functionId = anomalyFunctionDAO.save(getTestFunctionSpec(metric, collection));
// create test email configuration
emailConfigurationDAO.save(getTestEmailConfiguration(metric, collection));
// create test alert configuration
alertConfigDAO.save(getTestAlertConfiguration("test alert v2"));
// create test dataset config
datasetConfigDAO.save(getTestDatasetConfig(collection));
// setup function factory for worker and merger
InputStream factoryStream = AnomalyApplicationEndToEndTest.class.getResourceAsStream(functionPropertiesFile);
anomalyFunctionFactory = new AnomalyFunctionFactory(factoryStream);
// setup alertfilter factory for worker
InputStream alertFilterStream = AnomalyApplicationEndToEndTest.class.getResourceAsStream(alertFilterPropertiesFile);
alertFilterFactory = new AlertFilterFactory(alertFilterStream);
}
use of com.linkedin.pinot.client.ResultSetGroup in project pinot by linkedin.
the class PinotThirdEyeClient method execute.
@Override
public PinotThirdEyeResponse execute(ThirdEyeRequest request) throws Exception {
DatasetConfigDTO datasetConfig = CACHE_REGISTRY_INSTANCE.getDatasetConfigCache().get(request.getCollection());
TimeSpec dataTimeSpec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
List<MetricFunction> metricFunctions = request.getMetricFunctions();
List<String> dimensionNames = datasetConfig.getDimensions();
List<ResultSetGroup> resultSetGroups = new ArrayList<>();
// By default, query only offline, unless dataset has been marked as realtime
String tableName = ThirdEyeUtils.computeTableName(request.getCollection());
if (datasetConfig.isMetricAsDimension()) {
List<String> pqls = PqlUtils.getMetricAsDimensionPqls(request, dataTimeSpec, datasetConfig);
for (String pql : pqls) {
LOG.debug("PQL isMetricAsDimension : {}", pql);
ResultSetGroup result = CACHE_REGISTRY_INSTANCE.getResultSetGroupCache().get(new PinotQuery(pql, tableName));
resultSetGroups.add(result);
}
} else {
String sql = PqlUtils.getPql(request, dataTimeSpec);
LOG.debug("PQL: {}", sql);
ResultSetGroup result = CACHE_REGISTRY_INSTANCE.getResultSetGroupCache().get(new PinotQuery(sql, tableName));
resultSetGroups.add(result);
if (LOG.isDebugEnabled()) {
LOG.debug("Result for: {} {}", sql, format(result));
}
}
List<ResultSet> resultSets = getResultSets(resultSetGroups);
List<String[]> resultRows = parseResultSets(request, resultSets, metricFunctions, dimensionNames, datasetConfig);
PinotThirdEyeResponse resp = new PinotThirdEyeResponse(request, resultRows, dataTimeSpec);
return resp;
}
Aggregations