Examples with OfflineTableDataManager com.linkedin.pinot.core.data.manager.offline.OfflineTableDataManager used on opensource projects

Example 1 with OfflineTableDataManager

use of com.linkedin.pinot.core.data.manager.offline.OfflineTableDataManager in project pinot by linkedin.

the class OfflineTableDataManagerTest method basicTest.

@Test
public void basicTest() throws Exception {
    OfflineTableDataManager tableDataManager = makeTestableManager();
    final String segmentName = "TestSegment";
    final int totalDocs = 23456;
    // Add the segment, get it for use, remove the segment, and then return it.
    // Make sure that the segment is not destroyed before return.
    IndexSegment indexSegment = makeIndexSegment(segmentName, totalDocs);
    tableDataManager.addSegment(indexSegment);
    SegmentDataManager segmentDataManager = tableDataManager.acquireSegment(segmentName);
    verifyCount(segmentDataManager, 2);
    tableDataManager.removeSegment(segmentName);
    verifyCount(segmentDataManager, 1);
    Assert.assertEquals(_nDestroys, 0);
    tableDataManager.releaseSegment(segmentDataManager);
    verifyCount(segmentDataManager, 0);
    Assert.assertEquals(_nDestroys, 1);
    // Now the segment should not be available for use.Also, returning a null reader is fine
    segmentDataManager = tableDataManager.acquireSegment(segmentName);
    Assert.assertNull(segmentDataManager);
    ImmutableList<SegmentDataManager> segmentDataManagers = tableDataManager.acquireAllSegments();
    Assert.assertEquals(segmentDataManagers.size(), 0);
    tableDataManager.releaseSegment(segmentDataManager);
    // Removing the segment again is fine.
    tableDataManager.removeSegment(segmentName);
    // Add a new segment and remove it in order this time.
    final String anotherSeg = "AnotherSegment";
    IndexSegment ix1 = makeIndexSegment(anotherSeg, totalDocs);
    tableDataManager.addSegment(ix1);
    SegmentDataManager sdm1 = tableDataManager.acquireSegment(anotherSeg);
    verifyCount(sdm1, 2);
    // acquire all segments
    ImmutableList<SegmentDataManager> segmentDataManagersList = tableDataManager.acquireAllSegments();
    Assert.assertEquals(segmentDataManagersList.size(), 1);
    verifyCount(sdm1, 3);
    for (SegmentDataManager dataManager : segmentDataManagersList) {
        tableDataManager.releaseSegment(dataManager);
    }
    // count is back to original
    verifyCount(sdm1, 2);
    tableDataManager.releaseSegment(sdm1);
    verifyCount(sdm1, 1);
    // Now replace the segment with another one.
    IndexSegment ix2 = makeIndexSegment(anotherSeg, totalDocs + 1);
    tableDataManager.addSegment(ix2);
    // Now the previous one should have been destroyed, and
    verifyCount(sdm1, 0);
    verify(ix1, times(1)).destroy();
    // Delete ix2 without accessing it.
    SegmentDataManager sdm2 = _internalSegMap.get(anotherSeg);
    verifyCount(sdm2, 1);
    tableDataManager.removeSegment(anotherSeg);
    verifyCount(sdm2, 0);
    verify(ix2, times(1)).destroy();
    tableDataManager.shutDown();
}

Example 2 with OfflineTableDataManager

use of com.linkedin.pinot.core.data.manager.offline.OfflineTableDataManager in project pinot by linkedin.

the class OfflineTableDataManagerTest method runStressTest.

private void runStressTest(boolean replaceSegments) throws Exception {
    _lo = 0;
    // Total number of segments we have in the server.
    _hi = 30;
    final int numQueryThreads = 10;
    final int runTimeSec = 30;
    // With the current parameters, 3k ops take about 15 seconds, create about 90 segments and drop about half of them
    // Running with coverage, it provides complete coverage of the (relevant) lines in OfflineTableDataManager
    Random random = new Random();
    TableDataManager tableDataManager = makeTestableManager();
    for (int i = _lo; i <= _hi; i++) {
        final String segName = segmentPrefix + i;
        tableDataManager.addSegment(makeIndexSegment(segName, random.nextInt()));
        _allSegManagers.add(_internalSegMap.get(segName));
    }
    // replaces segments while online
    runStorageServer(numQueryThreads, runTimeSec, tableDataManager, replaceSegments);
    //    System.out.println("Nops = " + _numQueries + ",nDrops=" + _nDestroys + ",nCreates=" + _allSegments.size());
    tableDataManager.shutDown();
}

Example 3 with OfflineTableDataManager

use of com.linkedin.pinot.core.data.manager.offline.OfflineTableDataManager in project pinot by linkedin.

the class OfflineTableDataManagerTest method makeTestableManager.

private OfflineTableDataManager makeTestableManager() throws Exception {
    OfflineTableDataManager tableDataManager = new OfflineTableDataManager();
    TableDataManagerConfig config;
    {
        config = mock(TableDataManagerConfig.class);
        when(config.getTableName()).thenReturn(tableName);
        when(config.getDataDir()).thenReturn(_tmpDir.getAbsolutePath());
        when(config.getReadMode()).thenReturn(readMode.toString());
        when(config.getIndexLoadingConfigMetadata()).thenReturn(null);
    }
    tableDataManager.init(config, new ServerMetrics(new MetricsRegistry()), null);
    tableDataManager.start();
    Field segsMapField = AbstractTableDataManager.class.getDeclaredField("_segmentsMap");
    segsMapField.setAccessible(true);
    _internalSegMap = (Map<String, OfflineSegmentDataManager>) segsMapField.get(tableDataManager);
    return tableDataManager;
}