Examples with IndexSegment com.linkedin.pinot.core.indexsegment.IndexSegment used on opensource projects

Example 31 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class FixedNumOfSegmentsPerJobQueryPlannerImpl method computeQueryPlan.

@Override
public QueryPlan computeQueryPlan(BrokerRequest brokerRequest, List<IndexSegment> indexSegmentList) {
    QueryPlanCreator queryPlanCreator = new QueryPlanCreator(brokerRequest);
    List<IndexSegment> vertexSegmentList = new ArrayList<IndexSegment>();
    int i = 0;
    for (IndexSegment indexSegment : indexSegmentList) {
        vertexSegmentList.add(indexSegment);
        if ((++i) % _numSegmentsPerJob == 0) {
            queryPlanCreator.addJobVertexWithDependency(null, new JobVertex(vertexSegmentList));
            vertexSegmentList = new ArrayList<IndexSegment>();
        }
    }
    if (vertexSegmentList.size() > 0) {
        queryPlanCreator.addJobVertexWithDependency(null, new JobVertex(vertexSegmentList));
    }
    return queryPlanCreator.buildQueryPlan();
}

Example 32 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class BrokerRequestPreProcessor method rewriteFastHllColumnName.

/**
   * Rewrite 'fasthll' column name.
   *
   * @param indexSegments list of index segments.
   * @param aggregationsInfo list of aggregation info.
   */
private static void rewriteFastHllColumnName(List<IndexSegment> indexSegments, List<AggregationInfo> aggregationsInfo) {
    // Consistent check.
    for (AggregationInfo aggregationInfo : aggregationsInfo) {
        if (aggregationInfo.getAggregationType().equalsIgnoreCase("fasthll")) {
            String column = aggregationInfo.getAggregationParams().get("column").trim();
            boolean isFirstSegment = true;
            String firstSegmentName = null;
            String hllDerivedColumn = null;
            for (IndexSegment indexSegment : indexSegments) {
                SegmentMetadata segmentMetadata = indexSegment.getSegmentMetadata();
                if (isFirstSegment) {
                    // Use metadata from first index segment to perform rewrite.
                    isFirstSegment = false;
                    firstSegmentName = segmentMetadata.getName();
                    hllDerivedColumn = segmentMetadata.getDerivedColumn(column, MetricFieldSpec.DerivedMetricType.HLL);
                    if (hllDerivedColumn != null) {
                        aggregationInfo.getAggregationParams().put("column", hllDerivedColumn);
                    }
                } else {
                    // Perform consistency check on other index segments.
                    String hllDerivedColumnToCheck = segmentMetadata.getDerivedColumn(column, MetricFieldSpec.DerivedMetricType.HLL);
                    if (!Objects.equals(hllDerivedColumn, hllDerivedColumnToCheck)) {
                        throw new RuntimeException("Found inconsistency HLL derived column name. In segment " + firstSegmentName + ": " + hllDerivedColumn + "; In segment " + segmentMetadata.getName() + ": " + hllDerivedColumnToCheck);
                    }
                }
            }
        }
    }
}

Example 33 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class InstancePlanMakerImplV2 method makeInterSegmentPlan.

@Override
public Plan makeInterSegmentPlan(List<SegmentDataManager> segmentDataManagers, BrokerRequest brokerRequest, ExecutorService executorService, long timeOutMs) {
    // TODO: pass in List<IndexSegment> directly.
    List<IndexSegment> indexSegments = new ArrayList<>(segmentDataManagers.size());
    for (SegmentDataManager segmentDataManager : segmentDataManagers) {
        indexSegments.add(segmentDataManager.getSegment());
    }
    BrokerRequestPreProcessor.preProcess(indexSegments, brokerRequest);
    List<PlanNode> planNodes = new ArrayList<>();
    for (IndexSegment indexSegment : indexSegments) {
        planNodes.add(makeInnerSegmentPlan(indexSegment, brokerRequest));
    }
    CombinePlanNode combinePlanNode = new CombinePlanNode(planNodes, brokerRequest, executorService, timeOutMs);
    return new GlobalPlanImplV0(new InstanceResponsePlanNode(combinePlanNode));
}

Example 34 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class SegmentV1V2ToV3FormatConverterTest method testConvert.

@Test
public void testConvert() throws Exception {
    SegmentMetadataImpl beforeConversionMeta = new SegmentMetadataImpl(segmentDirectory);
    SegmentV1V2ToV3FormatConverter converter = new SegmentV1V2ToV3FormatConverter();
    converter.convert(segmentDirectory);
    File v3Location = SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3);
    Assert.assertTrue(v3Location.exists());
    Assert.assertTrue(v3Location.isDirectory());
    Assert.assertTrue(new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).exists());
    SegmentMetadataImpl metadata = new SegmentMetadataImpl(v3Location);
    Assert.assertEquals(metadata.getVersion(), SegmentVersion.v3.toString());
    Assert.assertTrue(new File(v3Location, V1Constants.SEGMENT_CREATION_META).exists());
    // Drop the star tree index file because it has invalid data
    new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).delete();
    new File(segmentDirectory, V1Constants.STAR_TREE_INDEX_FILE).delete();
    FileTime afterConversionTime = Files.getLastModifiedTime(v3Location.toPath());
    // verify that the segment loads correctly. This is necessary and sufficient
    // full proof way to ensure that segment is correctly translated
    IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap, v3LoadingConfig);
    Assert.assertNotNull(indexSegment);
    Assert.assertEquals(indexSegment.getSegmentName(), metadata.getName());
    Assert.assertEquals(SegmentVersion.v3, SegmentVersion.valueOf(indexSegment.getSegmentMetadata().getVersion()));
    FileTime afterLoadTime = Files.getLastModifiedTime(v3Location.toPath());
    Assert.assertEquals(afterConversionTime, afterLoadTime);
    // verify that SegmentMetadataImpl loaded from segmentDirectory correctly sets
    // metadata information after conversion. This has impacted us while loading
    // segments by triggering download. That's costly. That's also difficult to test
    Assert.assertFalse(new File(segmentDirectory, V1Constants.MetadataKeys.METADATA_FILE_NAME).exists());
    SegmentMetadataImpl metaAfterConversion = new SegmentMetadataImpl(segmentDirectory);
    Assert.assertNotNull(metaAfterConversion);
    Assert.assertFalse(metaAfterConversion.getCrc().equalsIgnoreCase(String.valueOf(Long.MIN_VALUE)));
    Assert.assertEquals(metaAfterConversion.getCrc(), beforeConversionMeta.getCrc());
    Assert.assertTrue(metaAfterConversion.getIndexCreationTime() != Long.MIN_VALUE);
    Assert.assertEquals(metaAfterConversion.getIndexCreationTime(), beforeConversionMeta.getIndexCreationTime());
}

Example 35 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class LoadersTest method testLoad.

@Test
public void testLoad() throws Exception {
    SegmentMetadataImpl originalMetadata = new SegmentMetadataImpl(segmentDirectory);
    Assert.assertEquals(originalMetadata.getSegmentVersion(), SegmentVersion.v1);
    // note: ordering of these two test blocks matters
    {
        // Explicitly pass v1 format since we will convert by default to v3
        IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap, v1LoadingConfig);
        Assert.assertEquals(indexSegment.getSegmentMetadata().getVersion(), originalMetadata.getVersion());
        Assert.assertFalse(SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3).exists());
    }
    {
        // with this code and converter in place, make sure we still load original version
        // by default. We require specific configuration for v3.
        IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap);
        Assert.assertEquals(indexSegment.getSegmentMetadata().getVersion(), SegmentVersion.v3.toString());
        Assert.assertTrue(SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3).exists());
    }
}