Examples with IndexSegment com.linkedin.pinot.core.indexsegment.IndexSegment used on opensource projects

Example 1 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class RealtimeTableDataManager method replaceLLSegment.

// Replace a committed segment.
public void replaceLLSegment(String segmentId) {
    try {
        IndexSegment segment = ColumnarSegmentLoader.load(new File(_indexDir, segmentId), _readMode, _indexLoadingConfigMetadata);
        addSegment(segment);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 2 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class FixedNumJobsQueryPlannerImpl method computeQueryPlan.

@Override
public QueryPlan computeQueryPlan(BrokerRequest brokerRequest, List<IndexSegment> indexSegmentList) {
    QueryPlanCreator queryPlanCreator = new QueryPlanCreator(brokerRequest);
    List<JobVertex> jobVertexList = new ArrayList<JobVertex>();
    for (int i = 0; i < _numJobs; ++i) {
        jobVertexList.add(new JobVertex(new ArrayList<IndexSegment>()));
    }
    int i = 0;
    for (IndexSegment indexSegment : indexSegmentList) {
        (jobVertexList.get(i++).getIndexSegmentList()).add(indexSegment);
        if (i == _numJobs) {
            i = 0;
        }
    }
    for (JobVertex jobVertex : jobVertexList) {
        queryPlanCreator.addJobVertexWithDependency(null, jobVertex);
    }
    return queryPlanCreator.buildQueryPlan();
}

Example 3 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class ParallelQueryPlannerImpl method computeQueryPlan.

@Override
public QueryPlan computeQueryPlan(BrokerRequest query, List<IndexSegment> indexSegmentList) {
    QueryPlanCreator queryPlanCreator = new QueryPlanCreator(query);
    for (IndexSegment indexSegment : indexSegmentList) {
        List<IndexSegment> vertexSegmentList = new ArrayList<IndexSegment>();
        vertexSegmentList.add(indexSegment);
        queryPlanCreator.addJobVertexWithDependency(null, new JobVertex(vertexSegmentList));
    }
    return queryPlanCreator.buildQueryPlan();
}

Example 4 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class OfflineTableDataManagerTest method basicTest.

@Test
public void basicTest() throws Exception {
    OfflineTableDataManager tableDataManager = makeTestableManager();
    final String segmentName = "TestSegment";
    final int totalDocs = 23456;
    // Add the segment, get it for use, remove the segment, and then return it.
    // Make sure that the segment is not destroyed before return.
    IndexSegment indexSegment = makeIndexSegment(segmentName, totalDocs);
    tableDataManager.addSegment(indexSegment);
    SegmentDataManager segmentDataManager = tableDataManager.acquireSegment(segmentName);
    verifyCount(segmentDataManager, 2);
    tableDataManager.removeSegment(segmentName);
    verifyCount(segmentDataManager, 1);
    Assert.assertEquals(_nDestroys, 0);
    tableDataManager.releaseSegment(segmentDataManager);
    verifyCount(segmentDataManager, 0);
    Assert.assertEquals(_nDestroys, 1);
    // Now the segment should not be available for use.Also, returning a null reader is fine
    segmentDataManager = tableDataManager.acquireSegment(segmentName);
    Assert.assertNull(segmentDataManager);
    ImmutableList<SegmentDataManager> segmentDataManagers = tableDataManager.acquireAllSegments();
    Assert.assertEquals(segmentDataManagers.size(), 0);
    tableDataManager.releaseSegment(segmentDataManager);
    // Removing the segment again is fine.
    tableDataManager.removeSegment(segmentName);
    // Add a new segment and remove it in order this time.
    final String anotherSeg = "AnotherSegment";
    IndexSegment ix1 = makeIndexSegment(anotherSeg, totalDocs);
    tableDataManager.addSegment(ix1);
    SegmentDataManager sdm1 = tableDataManager.acquireSegment(anotherSeg);
    verifyCount(sdm1, 2);
    // acquire all segments
    ImmutableList<SegmentDataManager> segmentDataManagersList = tableDataManager.acquireAllSegments();
    Assert.assertEquals(segmentDataManagersList.size(), 1);
    verifyCount(sdm1, 3);
    for (SegmentDataManager dataManager : segmentDataManagersList) {
        tableDataManager.releaseSegment(dataManager);
    }
    // count is back to original
    verifyCount(sdm1, 2);
    tableDataManager.releaseSegment(sdm1);
    verifyCount(sdm1, 1);
    // Now replace the segment with another one.
    IndexSegment ix2 = makeIndexSegment(anotherSeg, totalDocs + 1);
    tableDataManager.addSegment(ix2);
    // Now the previous one should have been destroyed, and
    verifyCount(sdm1, 0);
    verify(ix1, times(1)).destroy();
    // Delete ix2 without accessing it.
    SegmentDataManager sdm2 = _internalSegMap.get(anotherSeg);
    verifyCount(sdm2, 1);
    tableDataManager.removeSegment(anotherSeg);
    verifyCount(sdm2, 0);
    verify(ix2, times(1)).destroy();
    tableDataManager.shutDown();
}

Example 5 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class HllIndexCreationTest method testColumnStatsWithStarTree.

@Test
public void testColumnStatsWithStarTree() throws Exception {
    SegmentWithHllIndexCreateHelper helper = null;
    boolean hasException = false;
    int maxDocLength = 10000;
    try {
        LOGGER.debug("================ With StarTree ================");
        helper = new SegmentWithHllIndexCreateHelper("withStarTree", getClass().getClassLoader().getResource(AVRO_DATA), timeColumnName, timeUnit, "starTreeSegment");
        SegmentIndexCreationDriver driver = helper.build(true, hllConfig);
        LOGGER.debug("================ Cardinality ================");
        for (String name : helper.getSchema().getColumnNames()) {
            LOGGER.debug("* " + name + ": " + driver.getColumnStatisticsCollector(name).getCardinality());
        }
        LOGGER.debug("Loading ...");
        IndexSegment indexSegment = Loaders.IndexSegment.load(helper.getSegmentDirectory(), ReadMode.mmap);
        int[] docIdSet = new int[maxDocLength];
        for (int i = 0; i < maxDocLength; i++) {
            docIdSet[i] = i;
        }
        Map<String, BaseOperator> dataSourceMap = new HashMap<>();
        for (String column : indexSegment.getColumnNames()) {
            dataSourceMap.put(column, indexSegment.getDataSource(column));
        }
        DataBlockCache blockCache = new DataBlockCache(new DataFetcher(dataSourceMap));
        blockCache.initNewBlock(docIdSet, 0, maxDocLength);
        String[] strings = blockCache.getStringValueArrayForColumn("column1_hll");
        Assert.assertEquals(strings.length, maxDocLength);
        double[] ints = blockCache.getDoubleValueArrayForColumn("column1");
        Assert.assertEquals(ints.length, maxDocLength);
    } catch (Exception e) {
        hasException = true;
        LOGGER.error(e.getMessage());
    } finally {
        if (helper != null) {
            helper.cleanTempDir();
        }
        Assert.assertEquals(hasException, false);
    }
}