Examples with IndexSegment com.linkedin.pinot.core.indexsegment.IndexSegment used on opensource projects

Example 6 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class ParallelQueryPlannerImpl method computeQueryPlan.

@Override
public QueryPlan computeQueryPlan(BrokerRequest query, List<IndexSegment> indexSegmentList) {
    QueryPlanCreator queryPlanCreator = new QueryPlanCreator(query);
    for (IndexSegment indexSegment : indexSegmentList) {
        List<IndexSegment> vertexSegmentList = new ArrayList<IndexSegment>();
        vertexSegmentList.add(indexSegment);
        queryPlanCreator.addJobVertexWithDependency(null, new JobVertex(vertexSegmentList));
    }
    return queryPlanCreator.buildQueryPlan();
}

Example 7 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class OfflineTableDataManagerTest method basicTest.

@Test
public void basicTest() throws Exception {
    OfflineTableDataManager tableDataManager = makeTestableManager();
    final String segmentName = "TestSegment";
    final int totalDocs = 23456;
    // Add the segment, get it for use, remove the segment, and then return it.
    // Make sure that the segment is not destroyed before return.
    IndexSegment indexSegment = makeIndexSegment(segmentName, totalDocs);
    tableDataManager.addSegment(indexSegment);
    SegmentDataManager segmentDataManager = tableDataManager.acquireSegment(segmentName);
    verifyCount(segmentDataManager, 2);
    tableDataManager.removeSegment(segmentName);
    verifyCount(segmentDataManager, 1);
    Assert.assertEquals(_nDestroys, 0);
    tableDataManager.releaseSegment(segmentDataManager);
    verifyCount(segmentDataManager, 0);
    Assert.assertEquals(_nDestroys, 1);
    // Now the segment should not be available for use.Also, returning a null reader is fine
    segmentDataManager = tableDataManager.acquireSegment(segmentName);
    Assert.assertNull(segmentDataManager);
    ImmutableList<SegmentDataManager> segmentDataManagers = tableDataManager.acquireAllSegments();
    Assert.assertEquals(segmentDataManagers.size(), 0);
    tableDataManager.releaseSegment(segmentDataManager);
    // Removing the segment again is fine.
    tableDataManager.removeSegment(segmentName);
    // Add a new segment and remove it in order this time.
    final String anotherSeg = "AnotherSegment";
    IndexSegment ix1 = makeIndexSegment(anotherSeg, totalDocs);
    tableDataManager.addSegment(ix1);
    SegmentDataManager sdm1 = tableDataManager.acquireSegment(anotherSeg);
    verifyCount(sdm1, 2);
    // acquire all segments
    ImmutableList<SegmentDataManager> segmentDataManagersList = tableDataManager.acquireAllSegments();
    Assert.assertEquals(segmentDataManagersList.size(), 1);
    verifyCount(sdm1, 3);
    for (SegmentDataManager dataManager : segmentDataManagersList) {
        tableDataManager.releaseSegment(dataManager);
    }
    // count is back to original
    verifyCount(sdm1, 2);
    tableDataManager.releaseSegment(sdm1);
    verifyCount(sdm1, 1);
    // Now replace the segment with another one.
    IndexSegment ix2 = makeIndexSegment(anotherSeg, totalDocs + 1);
    tableDataManager.addSegment(ix2);
    // Now the previous one should have been destroyed, and
    verifyCount(sdm1, 0);
    verify(ix1, times(1)).destroy();
    // Delete ix2 without accessing it.
    SegmentDataManager sdm2 = _internalSegMap.get(anotherSeg);
    verifyCount(sdm2, 1);
    tableDataManager.removeSegment(anotherSeg);
    verifyCount(sdm2, 0);
    verify(ix2, times(1)).destroy();
    tableDataManager.shutDown();
}

Example 8 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class HllIndexCreationTest method testColumnStatsWithStarTree.

@Test
public void testColumnStatsWithStarTree() throws Exception {
    SegmentWithHllIndexCreateHelper helper = null;
    boolean hasException = false;
    int maxDocLength = 10000;
    try {
        LOGGER.debug("================ With StarTree ================");
        helper = new SegmentWithHllIndexCreateHelper("withStarTree", getClass().getClassLoader().getResource(AVRO_DATA), timeColumnName, timeUnit, "starTreeSegment");
        SegmentIndexCreationDriver driver = helper.build(true, hllConfig);
        LOGGER.debug("================ Cardinality ================");
        for (String name : helper.getSchema().getColumnNames()) {
            LOGGER.debug("* " + name + ": " + driver.getColumnStatisticsCollector(name).getCardinality());
        }
        LOGGER.debug("Loading ...");
        IndexSegment indexSegment = Loaders.IndexSegment.load(helper.getSegmentDirectory(), ReadMode.mmap);
        int[] docIdSet = new int[maxDocLength];
        for (int i = 0; i < maxDocLength; i++) {
            docIdSet[i] = i;
        }
        Map<String, BaseOperator> dataSourceMap = new HashMap<>();
        for (String column : indexSegment.getColumnNames()) {
            dataSourceMap.put(column, indexSegment.getDataSource(column));
        }
        DataBlockCache blockCache = new DataBlockCache(new DataFetcher(dataSourceMap));
        blockCache.initNewBlock(docIdSet, 0, maxDocLength);
        String[] strings = blockCache.getStringValueArrayForColumn("column1_hll");
        Assert.assertEquals(strings.length, maxDocLength);
        double[] ints = blockCache.getDoubleValueArrayForColumn("column1");
        Assert.assertEquals(ints.length, maxDocLength);
    } catch (Exception e) {
        hasException = true;
        LOGGER.error(e.getMessage());
    } finally {
        if (helper != null) {
            helper.cleanTempDir();
        }
        Assert.assertEquals(hasException, false);
    }
}

Example 9 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class HllIndexCreationTest method testConvert.

@Test
public void testConvert() throws Exception {
    SegmentWithHllIndexCreateHelper helper = null;
    try {
        helper = new SegmentWithHllIndexCreateHelper("testConvert", getClass().getClassLoader().getResource(AVRO_DATA), timeColumnName, timeUnit, "starTreeSegment");
        SegmentIndexCreationDriver driver = helper.build(true, hllConfig);
        File segmentDirectory = new File(helper.getIndexDir(), driver.getSegmentName());
        LOGGER.debug("Segment Directory: " + segmentDirectory.getAbsolutePath());
        SegmentV1V2ToV3FormatConverter converter = new SegmentV1V2ToV3FormatConverter();
        converter.convert(segmentDirectory);
        File v3Location = SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3);
        LOGGER.debug("v3Location: " + v3Location.getAbsolutePath());
        Assert.assertTrue(v3Location.exists());
        Assert.assertTrue(v3Location.isDirectory());
        Assert.assertTrue(new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).exists());
        SegmentMetadataImpl metadata = new SegmentMetadataImpl(v3Location);
        LOGGER.debug("metadata all columns: " + metadata.getAllColumns());
        Assert.assertEquals(metadata.getVersion(), SegmentVersion.v3.toString());
        Assert.assertTrue(new File(v3Location, V1Constants.SEGMENT_CREATION_META).exists());
        // Drop the star tree index file because it has invalid data
        // new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).delete();
        // new File(segmentDirectory, V1Constants.STAR_TREE_INDEX_FILE).delete();
        FileTime afterConversionTime = Files.getLastModifiedTime(v3Location.toPath());
        // verify that the segment loads correctly. This is necessary and sufficient
        // full proof way to ensure that segment is correctly translated
        IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap, v3LoadingConfig);
        Assert.assertNotNull(indexSegment);
        Assert.assertEquals(indexSegment.getSegmentName(), metadata.getName());
        Assert.assertEquals(SegmentVersion.v3, SegmentVersion.valueOf(indexSegment.getSegmentMetadata().getVersion()));
    } finally {
        if (helper != null) {
            helper.cleanTempDir();
        }
    }
}

Example 10 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class ColumnMetadataTest method testAllFieldsExceptCreatorName.

@Test
public void testAllFieldsExceptCreatorName() throws Exception {
    // Build the Segment metadata.
    SegmentGeneratorConfig config = CreateSegmentConfigWithoutCreator();
    SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
    driver.init(config);
    driver.build();
    // Load segment metadata.
    IndexSegment segment = Loaders.IndexSegment.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    SegmentMetadataImpl metadata = (SegmentMetadataImpl) segment.getSegmentMetadata();
    verifySegmentAfterLoading(metadata);
    // Make sure we get null for creator name.
    String creatorName = metadata.getCreatorName();
    Assert.assertEquals(creatorName, null);
}