Examples with IndexSegment com.linkedin.pinot.core.indexsegment.IndexSegment used on opensource projects

Example 16 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class BrokerReduceServiceTest method testDistinctCountQuery1.

@Test
public void testDistinctCountQuery1() {
    BrokerRequest brokerRequest = getDistinctCountQuery("dim1");
    QuerySource querySource = new QuerySource();
    querySource.setTableName("midas");
    brokerRequest.setQuerySource(querySource);
    InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest);
    instanceRequest.setSearchSegments(new ArrayList<String>());
    for (IndexSegment segment : _indexSegmentList) {
        instanceRequest.addToSearchSegments(segment.getSegmentName());
    }
    Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
    try {
        QueryRequest queryRequest = new QueryRequest(instanceRequest, TableDataManagerProvider.getServerMetrics());
        DataTable instanceResponse1 = _queryExecutor.processQuery(queryRequest, queryRunners);
        instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse1);
        DataTable instanceResponse2 = _queryExecutor.processQuery(queryRequest, queryRunners);
        instanceResponseMap.put(new ServerInstance("localhost:1111"), instanceResponse2);
        BrokerResponseNative brokerResponse = _reduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
        AggregationResult aggregationResult = brokerResponse.getAggregationResults().get(0);
        LOGGER.info("BrokerResponse is " + aggregationResult);
        checkAggregationResult(aggregationResult, "distinctCount_dim1", 100.0);
        LOGGER.info("Time used for BrokerResponse is " + brokerResponse.getTimeUsedMs());
    } catch (Exception e) {
        e.printStackTrace();
        // Should never happen
        throw new RuntimeException(e.toString(), e);
    }
}

Example 17 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class QueryExecutorTest method testSumQuery.

@Test
public void testSumQuery() {
    BrokerRequest brokerRequest = getSumQuery();
    QuerySource querySource = new QuerySource();
    querySource.setTableName("midas");
    brokerRequest.setQuerySource(querySource);
    InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest);
    instanceRequest.setSearchSegments(new ArrayList<String>());
    for (IndexSegment segment : _indexSegmentList) {
        instanceRequest.getSearchSegments().add(segment.getSegmentName());
    }
    QueryRequest queryRequest = new QueryRequest(instanceRequest, serverMetrics);
    DataTable instanceResponse = _queryExecutor.processQuery(queryRequest, queryRunners);
    LOGGER.info("InstanceResponse is " + instanceResponse.getDouble(0, 0));
    Assert.assertEquals(instanceResponse.getDouble(0, 0), 40000200000.0);
    LOGGER.info("Time used for instanceResponse is " + instanceResponse.getMetadata().get(DataTable.TIME_USED_MS_METADATA_KEY));
}

Example 18 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class QueryExecutorTest method testMinQuery.

@Test
public void testMinQuery() {
    BrokerRequest brokerRequest = getMinQuery();
    QuerySource querySource = new QuerySource();
    querySource.setTableName("midas");
    brokerRequest.setQuerySource(querySource);
    InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest);
    instanceRequest.setSearchSegments(new ArrayList<String>());
    for (IndexSegment segment : _indexSegmentList) {
        instanceRequest.getSearchSegments().add(segment.getSegmentName());
    }
    QueryRequest queryRequest = new QueryRequest(instanceRequest, serverMetrics);
    DataTable instanceResponse = _queryExecutor.processQuery(queryRequest, queryRunners);
    LOGGER.info("InstanceResponse is " + instanceResponse.getDouble(0, 0));
    Assert.assertEquals(instanceResponse.getDouble(0, 0), 0.0);
    LOGGER.info("Time used for instanceResponse is " + instanceResponse.getMetadata().get(DataTable.TIME_USED_MS_METADATA_KEY));
}

Example 19 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class BrokerReduceServiceTest method testAvgQuery.

@Test
public void testAvgQuery() {
    BrokerRequest brokerRequest = getAvgQuery();
    QuerySource querySource = new QuerySource();
    querySource.setTableName("midas");
    brokerRequest.setQuerySource(querySource);
    InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest);
    instanceRequest.setSearchSegments(new ArrayList<String>());
    for (IndexSegment segment : _indexSegmentList) {
        instanceRequest.addToSearchSegments(segment.getSegmentName());
    }
    Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
    try {
        QueryRequest queryRequest = new QueryRequest(instanceRequest, TableDataManagerProvider.getServerMetrics());
        DataTable instanceResponse1 = _queryExecutor.processQuery(queryRequest, queryRunners);
        instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse1);
        DataTable instanceResponse2 = _queryExecutor.processQuery(queryRequest, queryRunners);
        instanceResponseMap.put(new ServerInstance("localhost:1111"), instanceResponse2);
        BrokerResponseNative brokerResponse = _reduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
        AggregationResult aggregationResult = brokerResponse.getAggregationResults().get(0);
        LOGGER.info("BrokerResponse is " + aggregationResult);
        checkAggregationResult(aggregationResult, "avg_met", 100000.0);
        LOGGER.info("Time used for BrokerResponse is " + brokerResponse.getTimeUsedMs());
    } catch (Exception e) {
        e.printStackTrace();
        // Should never happen
        throw new RuntimeException(e.toString(), e);
    }
}

Example 20 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class TableSizeResource method getTableSize.

@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/tables/{tableName}/size")
@ApiOperation(value = "Show table storage size", notes = "Lists size of all the segments of the table")
@ApiResponses(value = { @ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 500, message = "Internal server error"), @ApiResponse(code = 404, message = "Table not found") })
public TableSizeInfo getTableSize(@ApiParam(value = "Table Name with type", required = true) @PathParam("tableName") String tableName, @ApiParam(value = "Provide detailed information", required = false) @DefaultValue("true") @QueryParam("detailed") boolean detailed) throws WebApplicationException {
    InstanceDataManager dataManager = (InstanceDataManager) serverInstance.getInstanceDataManager();
    if (dataManager == null) {
        throw new WebApplicationException("Invalid server initialization", Response.Status.INTERNAL_SERVER_ERROR);
    }
    TableDataManager tableDataManager = dataManager.getTableDataManager(tableName);
    if (tableDataManager == null) {
        throw new WebApplicationException("Table: " + tableName + " is not found", Response.Status.NOT_FOUND);
    }
    TableSizeInfo tableSizeInfo = new TableSizeInfo();
    tableSizeInfo.tableName = tableDataManager.getTableName();
    tableSizeInfo.diskSizeInBytes = 0L;
    ImmutableList<SegmentDataManager> segmentDataManagers = tableDataManager.acquireAllSegments();
    try {
        for (SegmentDataManager segmentDataManager : segmentDataManagers) {
            IndexSegment segment = segmentDataManager.getSegment();
            long segmentSizeBytes = segment.getDiskSizeBytes();
            if (detailed) {
                SegmentSizeInfo segmentSizeInfo = new SegmentSizeInfo(segment.getSegmentName(), segmentSizeBytes);
                tableSizeInfo.segments.add(segmentSizeInfo);
            }
            tableSizeInfo.diskSizeInBytes += segmentSizeBytes;
        }
    } finally {
        // executes fast so duration of holding segments is not a concern
        for (SegmentDataManager segmentDataManager : segmentDataManagers) {
            tableDataManager.releaseSegment(segmentDataManager);
        }
    }
    //invalid to use the segmentDataManagers below
    return tableSizeInfo;
}