Examples with SegmentDataManager com.linkedin.pinot.core.data.manager.offline.SegmentDataManager used on opensource projects

Example 1 with SegmentDataManager

use of com.linkedin.pinot.core.data.manager.offline.SegmentDataManager in project pinot by linkedin.

the class RealtimeTableDataManager method doShutdown.

@Override
protected void doShutdown() {
    _segmentAsyncExecutorService.shutdown();
    for (SegmentDataManager segmentDataManager : _segmentsMap.values()) {
        segmentDataManager.destroy();
    }
    KafkaConsumerManager.closeAllConsumers();
}

Example 2 with SegmentDataManager

use of com.linkedin.pinot.core.data.manager.offline.SegmentDataManager in project pinot by linkedin.

the class OfflineTableDataManagerTest method runStorageServer.

private void runStorageServer(int numQueryThreads, int runTimeSec, TableDataManager tableDataManager, boolean replaceSegments) throws Exception {
    // Start 1 helix worker thread and as many query threads as configured.
    List<Thread> queryThreads = new ArrayList<>(numQueryThreads);
    for (int i = 0; i < numQueryThreads; i++) {
        TestSegmentUser segUser = new TestSegmentUser(tableDataManager);
        Thread segUserThread = new Thread(segUser);
        queryThreads.add(segUserThread);
        segUserThread.start();
    }
    TestHelixWorker helixWorker = new TestHelixWorker(tableDataManager, replaceSegments);
    Thread helixWorkerThread = new Thread(helixWorker);
    helixWorkerThread.start();
    _masterThread = Thread.currentThread();
    try {
        Thread.sleep(runTimeSec * 1000);
    } catch (InterruptedException e) {
    }
    _closing = true;
    helixWorkerThread.join();
    for (Thread t : queryThreads) {
        t.join();
    }
    if (_exception != null) {
        Assert.fail("One of the threads failed", _exception);
    }
    // tableDataManager should be quiescent now.
    // All segments we ever created must have a corresponding segment manager.
    Assert.assertEquals(_allSegManagers.size(), _allSegments.size());
    final int nSegsAcccessed = _accessedSegManagers.size();
    for (SegmentDataManager segmentDataManager : _internalSegMap.values()) {
        verifyCount(segmentDataManager, 1);
        // We should never have called destroy on these segments. Remove it from the list of accessed segments.
        verify(segmentDataManager.getSegment(), never()).destroy();
        _allSegManagers.remove(segmentDataManager);
        _accessedSegManagers.remove(segmentDataManager);
    }
    // For the remaining segments in accessed list, destroy must have been called exactly once.
    for (SegmentDataManager segmentDataManager : _allSegManagers) {
        verify(segmentDataManager.getSegment(), times(1)).destroy();
        // Also their count should be 0
        verifyCount(segmentDataManager, 0);
    }
    // The number of segments we accessed must be <= total segments created.
    Assert.assertTrue(nSegsAcccessed <= _allSegments.size(), "Accessed=" + nSegsAcccessed + ",created=" + _allSegments.size());
    // The number of segments we have seen and that are not there anymore, must be <= number destroyed.
    Assert.assertTrue(_accessedSegManagers.size() <= _nDestroys, "SeenButUnavailableNow=" + _accessedSegManagers.size() + ",Destroys=" + _nDestroys);
    // The current number of segments must be the as expected (hi-lo+1)
    Assert.assertEquals(_internalSegMap.size(), _hi - _lo + 1);
}

Example 3 with SegmentDataManager

use of com.linkedin.pinot.core.data.manager.offline.SegmentDataManager in project pinot by linkedin.

the class OfflineTableDataManagerTest method basicTest.

@Test
public void basicTest() throws Exception {
    OfflineTableDataManager tableDataManager = makeTestableManager();
    final String segmentName = "TestSegment";
    final int totalDocs = 23456;
    // Add the segment, get it for use, remove the segment, and then return it.
    // Make sure that the segment is not destroyed before return.
    IndexSegment indexSegment = makeIndexSegment(segmentName, totalDocs);
    tableDataManager.addSegment(indexSegment);
    SegmentDataManager segmentDataManager = tableDataManager.acquireSegment(segmentName);
    verifyCount(segmentDataManager, 2);
    tableDataManager.removeSegment(segmentName);
    verifyCount(segmentDataManager, 1);
    Assert.assertEquals(_nDestroys, 0);
    tableDataManager.releaseSegment(segmentDataManager);
    verifyCount(segmentDataManager, 0);
    Assert.assertEquals(_nDestroys, 1);
    // Now the segment should not be available for use.Also, returning a null reader is fine
    segmentDataManager = tableDataManager.acquireSegment(segmentName);
    Assert.assertNull(segmentDataManager);
    ImmutableList<SegmentDataManager> segmentDataManagers = tableDataManager.acquireAllSegments();
    Assert.assertEquals(segmentDataManagers.size(), 0);
    tableDataManager.releaseSegment(segmentDataManager);
    // Removing the segment again is fine.
    tableDataManager.removeSegment(segmentName);
    // Add a new segment and remove it in order this time.
    final String anotherSeg = "AnotherSegment";
    IndexSegment ix1 = makeIndexSegment(anotherSeg, totalDocs);
    tableDataManager.addSegment(ix1);
    SegmentDataManager sdm1 = tableDataManager.acquireSegment(anotherSeg);
    verifyCount(sdm1, 2);
    // acquire all segments
    ImmutableList<SegmentDataManager> segmentDataManagersList = tableDataManager.acquireAllSegments();
    Assert.assertEquals(segmentDataManagersList.size(), 1);
    verifyCount(sdm1, 3);
    for (SegmentDataManager dataManager : segmentDataManagersList) {
        tableDataManager.releaseSegment(dataManager);
    }
    // count is back to original
    verifyCount(sdm1, 2);
    tableDataManager.releaseSegment(sdm1);
    verifyCount(sdm1, 1);
    // Now replace the segment with another one.
    IndexSegment ix2 = makeIndexSegment(anotherSeg, totalDocs + 1);
    tableDataManager.addSegment(ix2);
    // Now the previous one should have been destroyed, and
    verifyCount(sdm1, 0);
    verify(ix1, times(1)).destroy();
    // Delete ix2 without accessing it.
    SegmentDataManager sdm2 = _internalSegMap.get(anotherSeg);
    verifyCount(sdm2, 1);
    tableDataManager.removeSegment(anotherSeg);
    verifyCount(sdm2, 0);
    verify(ix2, times(1)).destroy();
    tableDataManager.shutDown();
}

Example 4 with SegmentDataManager

use of com.linkedin.pinot.core.data.manager.offline.SegmentDataManager in project pinot by linkedin.

the class TableSizeResource method getTableSize.

@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/tables/{tableName}/size")
@ApiOperation(value = "Show table storage size", notes = "Lists size of all the segments of the table")
@ApiResponses(value = { @ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 500, message = "Internal server error"), @ApiResponse(code = 404, message = "Table not found") })
public TableSizeInfo getTableSize(@ApiParam(value = "Table Name with type", required = true) @PathParam("tableName") String tableName, @ApiParam(value = "Provide detailed information", required = false) @DefaultValue("true") @QueryParam("detailed") boolean detailed) throws WebApplicationException {
    InstanceDataManager dataManager = (InstanceDataManager) serverInstance.getInstanceDataManager();
    if (dataManager == null) {
        throw new WebApplicationException("Invalid server initialization", Response.Status.INTERNAL_SERVER_ERROR);
    }
    TableDataManager tableDataManager = dataManager.getTableDataManager(tableName);
    if (tableDataManager == null) {
        throw new WebApplicationException("Table: " + tableName + " is not found", Response.Status.NOT_FOUND);
    }
    TableSizeInfo tableSizeInfo = new TableSizeInfo();
    tableSizeInfo.tableName = tableDataManager.getTableName();
    tableSizeInfo.diskSizeInBytes = 0L;
    ImmutableList<SegmentDataManager> segmentDataManagers = tableDataManager.acquireAllSegments();
    try {
        for (SegmentDataManager segmentDataManager : segmentDataManagers) {
            IndexSegment segment = segmentDataManager.getSegment();
            long segmentSizeBytes = segment.getDiskSizeBytes();
            if (detailed) {
                SegmentSizeInfo segmentSizeInfo = new SegmentSizeInfo(segment.getSegmentName(), segmentSizeBytes);
                tableSizeInfo.segments.add(segmentSizeInfo);
            }
            tableSizeInfo.diskSizeInBytes += segmentSizeBytes;
        }
    } finally {
        // executes fast so duration of holding segments is not a concern
        for (SegmentDataManager segmentDataManager : segmentDataManagers) {
            tableDataManager.releaseSegment(segmentDataManager);
        }
    }
    //invalid to use the segmentDataManagers below
    return tableSizeInfo;
}

Example 5 with SegmentDataManager

use of com.linkedin.pinot.core.data.manager.offline.SegmentDataManager in project pinot by linkedin.

the class TablesResource method listTableSegments.

@GET
@Path("/tables/{tableName}/segments")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "List table segments", notes = "List segments of table hosted on this server")
@ApiResponses(value = { @ApiResponse(code = 200, message = "Success", response = TableSegments.class), @ApiResponse(code = 500, message = "Server initialization error", response = ErrorInfo.class) })
public TableSegments listTableSegments(@ApiParam(value = "Table name including type", required = true, example = "myTable_OFFLINE") @PathParam("tableName") String tableName) {
    TableDataManager tableDataManager = checkGetTableDataManager(tableName);
    ImmutableList<SegmentDataManager> segmentDataManagers = tableDataManager.acquireAllSegments();
    List<String> segments = new ArrayList<>(segmentDataManagers.size());
    for (SegmentDataManager segmentDataManager : segmentDataManagers) {
        segments.add(segmentDataManager.getSegmentName());
        tableDataManager.releaseSegment(segmentDataManager);
    }
    return new TableSegments(segments);
}