Examples with TableSizeInfo com.linkedin.pinot.common.restlet.resources.TableSizeInfo used on opensource projects

Example 1 with TableSizeInfo

use of com.linkedin.pinot.common.restlet.resources.TableSizeInfo in project pinot by linkedin.

the class ServerTableSizeReaderTest method createTableSizeInfo.

private TableSizeInfo createTableSizeInfo(String tableName, List<Integer> segmentIndexes) {
    TableSizeInfo tableSizeInfo = new TableSizeInfo();
    tableSizeInfo.tableName = tableName;
    tableSizeInfo.diskSizeInBytes = 0;
    for (int segmentIndex : segmentIndexes) {
        long size = segmentIndexToSize(segmentIndex);
        tableSizeInfo.diskSizeInBytes += size;
        SegmentSizeInfo s = new SegmentSizeInfo("seg" + segmentIndex, size);
        tableSizeInfo.segments.add(s);
    }
    return tableSizeInfo;
}

Example 2 with TableSizeInfo

use of com.linkedin.pinot.common.restlet.resources.TableSizeInfo in project pinot by linkedin.

the class TableSizeResource method getTableSize.

@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/tables/{tableName}/size")
@ApiOperation(value = "Show table storage size", notes = "Lists size of all the segments of the table")
@ApiResponses(value = { @ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 500, message = "Internal server error"), @ApiResponse(code = 404, message = "Table not found") })
public TableSizeInfo getTableSize(@ApiParam(value = "Table Name with type", required = true) @PathParam("tableName") String tableName, @ApiParam(value = "Provide detailed information", required = false) @DefaultValue("true") @QueryParam("detailed") boolean detailed) throws WebApplicationException {
    InstanceDataManager dataManager = (InstanceDataManager) serverInstance.getInstanceDataManager();
    if (dataManager == null) {
        throw new WebApplicationException("Invalid server initialization", Response.Status.INTERNAL_SERVER_ERROR);
    }
    TableDataManager tableDataManager = dataManager.getTableDataManager(tableName);
    if (tableDataManager == null) {
        throw new WebApplicationException("Table: " + tableName + " is not found", Response.Status.NOT_FOUND);
    }
    TableSizeInfo tableSizeInfo = new TableSizeInfo();
    tableSizeInfo.tableName = tableDataManager.getTableName();
    tableSizeInfo.diskSizeInBytes = 0L;
    ImmutableList<SegmentDataManager> segmentDataManagers = tableDataManager.acquireAllSegments();
    try {
        for (SegmentDataManager segmentDataManager : segmentDataManagers) {
            IndexSegment segment = segmentDataManager.getSegment();
            long segmentSizeBytes = segment.getDiskSizeBytes();
            if (detailed) {
                SegmentSizeInfo segmentSizeInfo = new SegmentSizeInfo(segment.getSegmentName(), segmentSizeBytes);
                tableSizeInfo.segments.add(segmentSizeInfo);
            }
            tableSizeInfo.diskSizeInBytes += segmentSizeBytes;
        }
    } finally {
        // executes fast so duration of holding segments is not a concern
        for (SegmentDataManager segmentDataManager : segmentDataManagers) {
            tableDataManager.releaseSegment(segmentDataManager);
        }
    }
    //invalid to use the segmentDataManagers below
    return tableSizeInfo;
}

Example 3 with TableSizeInfo

use of com.linkedin.pinot.common.restlet.resources.TableSizeInfo in project pinot by linkedin.

the class TableSizeResourceTest method testTableSizeDetailed.

@Test
public void testTableSizeDetailed() {
    TableSizeInfo tableSizeInfo = target.path(TABLE_SIZE_PATH).request().get(TableSizeInfo.class);
    IndexSegment indexSegment = testHelper.indexSegment;
    Assert.assertEquals(tableSizeInfo.tableName, TABLE_NAME);
    Assert.assertEquals(tableSizeInfo.diskSizeInBytes, indexSegment.getDiskSizeBytes());
    Assert.assertEquals(tableSizeInfo.segments.size(), 1);
    Assert.assertEquals(tableSizeInfo.segments.get(0).segmentName, indexSegment.getSegmentName());
    Assert.assertEquals(tableSizeInfo.segments.get(0).diskSizeInBytes, indexSegment.getDiskSizeBytes());
    Assert.assertEquals(tableSizeInfo.diskSizeInBytes, indexSegment.getDiskSizeBytes());
}

Example 4 with TableSizeInfo

use of com.linkedin.pinot.common.restlet.resources.TableSizeInfo in project pinot by linkedin.

the class ServerTableSizeReader method getSizeDetailsFromServers.

public Map<String, List<SegmentSizeInfo>> getSizeDetailsFromServers(BiMap<String, String> serverEndPoints, String table, int timeoutMsec) {
    List<String> serverUrls = new ArrayList<>(serverEndPoints.size());
    BiMap<String, String> endpointsToServers = serverEndPoints.inverse();
    for (String endpoint : endpointsToServers.keySet()) {
        String tableSizeUri = "http://" + endpoint + "/table/" + table + "/size";
        serverUrls.add(tableSizeUri);
    }
    MultiGetRequest mget = new MultiGetRequest(executor, connectionManager);
    LOGGER.info("Reading segment sizes from servers for table: {}, timeoutMsec: {}", table, timeoutMsec);
    CompletionService<GetMethod> completionService = mget.execute(serverUrls, timeoutMsec);
    Map<String, List<SegmentSizeInfo>> serverSegmentSizes = new HashMap<>(serverEndPoints.size());
    for (int i = 0; i < serverUrls.size(); i++) {
        GetMethod getMethod = null;
        try {
            getMethod = completionService.take().get();
            URI uri = getMethod.getURI();
            String instance = endpointsToServers.get(uri.getHost() + ":" + uri.getPort());
            if (getMethod.getStatusCode() >= 300) {
                LOGGER.error("Server: {} returned error: {}", instance, getMethod.getStatusCode());
                continue;
            }
            TableSizeInfo tableSizeInfo = new ObjectMapper().readValue(getMethod.getResponseBodyAsString(), TableSizeInfo.class);
            serverSegmentSizes.put(instance, tableSizeInfo.segments);
        } catch (InterruptedException e) {
            LOGGER.warn("Interrupted exception while reading segment size for table: {}", table, e);
        } catch (ExecutionException e) {
            if (Throwables.getRootCause(e) instanceof SocketTimeoutException) {
                LOGGER.warn("Server request to read table size was timed out for table: {}", table, e);
            } else if (Throwables.getRootCause(e) instanceof ConnectTimeoutException) {
                LOGGER.warn("Server request to read table size timed out waiting for connection. table: {}", table, e);
            } else if (Throwables.getRootCause(e) instanceof ConnectionPoolTimeoutException) {
                LOGGER.warn("Server request to read table size timed out on getting a connection from pool, table: {}", table, e);
            } else {
                LOGGER.warn("Execution exception while reading segment sizes for table: {}", table, e);
            }
        } catch (Exception e) {
            LOGGER.warn("Error while reading segment sizes for table: {}", table);
        } finally {
            if (getMethod != null) {
                getMethod.releaseConnection();
            }
        }
    }
    LOGGER.info("Finished reading segment sizes for table: {}", table);
    return serverSegmentSizes;
}

Example 5 with TableSizeInfo

use of com.linkedin.pinot.common.restlet.resources.TableSizeInfo in project pinot by linkedin.

the class TableSizeReaderTest method createHandler.

private HttpHandler createHandler(final int status, final List<SegmentSizeInfo> segmentSizes, final int sleepTimeMs) {
    return new HttpHandler() {

        @Override
        public void handle(HttpExchange httpExchange) throws IOException {
            if (sleepTimeMs > 0) {
                try {
                    Thread.sleep(sleepTimeMs);
                } catch (InterruptedException e) {
                    LOGGER.info("Handler interrupted during sleep");
                }
            }
            TableSizeInfo tableInfo = new TableSizeInfo("myTable", 0);
            tableInfo.segments = segmentSizes;
            for (SegmentSizeInfo segmentSize : segmentSizes) {
                tableInfo.diskSizeInBytes += segmentSize.diskSizeInBytes;
            }
            String json = new ObjectMapper().writeValueAsString(tableInfo);
            httpExchange.sendResponseHeaders(status, json.length());
            OutputStream responseBody = httpExchange.getResponseBody();
            responseBody.write(json.getBytes());
            responseBody.close();
        }
    };
}