Examples with IndexLoadingConfigMetadata com.linkedin.pinot.common.metadata.segment.IndexLoadingConfigMetadata used on opensource projects

Example 6 with IndexLoadingConfigMetadata

use of com.linkedin.pinot.common.metadata.segment.IndexLoadingConfigMetadata in project pinot by linkedin.

the class BitmapIndexCreationBenchmark method main.

public static void main(String[] args) throws Exception {
    System.out.println("Starting generation");
    Configuration tableDataManagerConfig = new PropertiesConfiguration();
    List<String> indexColumns = Lists.newArrayList("contract_id", "seat_id");
    tableDataManagerConfig.setProperty(IndexLoadingConfigMetadata.KEY_OF_LOADING_INVERTED_INDEX, indexColumns);
    IndexLoadingConfigMetadata indexLoadingConfigMetadata = new IndexLoadingConfigMetadata(tableDataManagerConfig);
    ReadMode mode = ReadMode.heap;
    File indexDir = new File("/home/kgopalak/pinot_perf/index_dir/capReportingEvents_OFFLINE/capReportingEvents_capReportingEvents_daily_2");
    long start = System.currentTimeMillis();
    Loaders.IndexSegment.load(indexDir, mode, indexLoadingConfigMetadata);
    long end = System.currentTimeMillis();
    System.out.println("Took " + (end - start) + " to generate bitmap index");
}

Example 7 with IndexLoadingConfigMetadata

use of com.linkedin.pinot.common.metadata.segment.IndexLoadingConfigMetadata in project pinot by linkedin.

the class IndexLoadingConfigMetadataTest method testEnableDefaultColumnsConfig.

@Test
public void testEnableDefaultColumnsConfig() {
    Configuration resourceMetadata = getTestResourceMetadata();
    IndexLoadingConfigMetadata indexLoadingConfigMetadata = new IndexLoadingConfigMetadata(resourceMetadata);
    Assert.assertTrue(indexLoadingConfigMetadata.isEnableDefaultColumns());
}

Example 8 with IndexLoadingConfigMetadata

use of com.linkedin.pinot.common.metadata.segment.IndexLoadingConfigMetadata in project pinot by linkedin.

the class HllIndexCreationTest method setUp.

@BeforeMethod
public void setUp() throws Exception {
    hllConfig = new HllConfig(hllLog2m, columnsToDeriveHllFields, hllDeriveColumnSuffix);
    Configuration tableConfig = new PropertiesConfiguration();
    tableConfig.addProperty(IndexLoadingConfigMetadata.KEY_OF_SEGMENT_FORMAT_VERSION, "v1");
    v1LoadingConfig = new IndexLoadingConfigMetadata(tableConfig);
    tableConfig.clear();
    tableConfig.addProperty(IndexLoadingConfigMetadata.KEY_OF_SEGMENT_FORMAT_VERSION, "v3");
    v3LoadingConfig = new IndexLoadingConfigMetadata(tableConfig);
}

Example 9 with IndexLoadingConfigMetadata

use of com.linkedin.pinot.common.metadata.segment.IndexLoadingConfigMetadata in project pinot by linkedin.

the class SegmentPreProcessorTest method testAddColumnMinMaxValue.

@Test
public void testAddColumnMinMaxValue() throws Exception {
    constructSegment();
    IndexLoadingConfigMetadata indexLoadingConfigMetadata = new IndexLoadingConfigMetadata(new PropertiesConfiguration());
    indexLoadingConfigMetadata.setGenerateColumnMinMaxValueMode(ColumnMinMaxValueGeneratorMode.NONE.toString());
    SegmentPreProcessor processor = new SegmentPreProcessor(_segmentDirectoryFile, indexLoadingConfigMetadata, null);
    processor.process();
    SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(_segmentDirectoryFile);
    ColumnMetadata timeColumnMetadata = segmentMetadata.getColumnMetadataFor("daysSinceEpoch");
    ColumnMetadata dimensionColumnMetadata = segmentMetadata.getColumnMetadataFor("column1");
    ColumnMetadata metricColumnMetadata = segmentMetadata.getColumnMetadataFor("count");
    Assert.assertNull(timeColumnMetadata.getMinValue());
    Assert.assertNull(timeColumnMetadata.getMaxValue());
    Assert.assertNull(dimensionColumnMetadata.getMinValue());
    Assert.assertNull(dimensionColumnMetadata.getMaxValue());
    Assert.assertNull(metricColumnMetadata.getMinValue());
    Assert.assertNull(metricColumnMetadata.getMaxValue());
    indexLoadingConfigMetadata = new IndexLoadingConfigMetadata(new PropertiesConfiguration());
    indexLoadingConfigMetadata.setGenerateColumnMinMaxValueMode(ColumnMinMaxValueGeneratorMode.TIME.toString());
    processor = new SegmentPreProcessor(_segmentDirectoryFile, indexLoadingConfigMetadata, null);
    processor.process();
    segmentMetadata = new SegmentMetadataImpl(_segmentDirectoryFile);
    timeColumnMetadata = segmentMetadata.getColumnMetadataFor("daysSinceEpoch");
    dimensionColumnMetadata = segmentMetadata.getColumnMetadataFor("column5");
    metricColumnMetadata = segmentMetadata.getColumnMetadataFor("count");
    Assert.assertEquals(timeColumnMetadata.getMinValue(), 1756015683);
    Assert.assertEquals(timeColumnMetadata.getMaxValue(), 1756015683);
    Assert.assertNull(dimensionColumnMetadata.getMinValue());
    Assert.assertNull(dimensionColumnMetadata.getMaxValue());
    Assert.assertNull(metricColumnMetadata.getMinValue());
    Assert.assertNull(metricColumnMetadata.getMaxValue());
    indexLoadingConfigMetadata.setGenerateColumnMinMaxValueMode(ColumnMinMaxValueGeneratorMode.NON_METRIC.toString());
    processor = new SegmentPreProcessor(_segmentDirectoryFile, indexLoadingConfigMetadata, null);
    processor.process();
    segmentMetadata = new SegmentMetadataImpl(_segmentDirectoryFile);
    timeColumnMetadata = segmentMetadata.getColumnMetadataFor("daysSinceEpoch");
    dimensionColumnMetadata = segmentMetadata.getColumnMetadataFor("column5");
    metricColumnMetadata = segmentMetadata.getColumnMetadataFor("count");
    Assert.assertEquals(timeColumnMetadata.getMinValue(), 1756015683);
    Assert.assertEquals(timeColumnMetadata.getMaxValue(), 1756015683);
    Assert.assertEquals(dimensionColumnMetadata.getMinValue(), "AKXcXcIqsqOJFsdwxZ");
    Assert.assertEquals(dimensionColumnMetadata.getMaxValue(), "yQkJTLOQoOqqhkAClgC");
    Assert.assertNull(metricColumnMetadata.getMinValue());
    Assert.assertNull(metricColumnMetadata.getMaxValue());
    indexLoadingConfigMetadata.setGenerateColumnMinMaxValueMode(ColumnMinMaxValueGeneratorMode.ALL.toString());
    processor = new SegmentPreProcessor(_segmentDirectoryFile, indexLoadingConfigMetadata, null);
    processor.process();
    segmentMetadata = new SegmentMetadataImpl(_segmentDirectoryFile);
    timeColumnMetadata = segmentMetadata.getColumnMetadataFor("daysSinceEpoch");
    dimensionColumnMetadata = segmentMetadata.getColumnMetadataFor("column5");
    metricColumnMetadata = segmentMetadata.getColumnMetadataFor("count");
    Assert.assertEquals(timeColumnMetadata.getMinValue(), 1756015683);
    Assert.assertEquals(timeColumnMetadata.getMaxValue(), 1756015683);
    Assert.assertEquals(dimensionColumnMetadata.getMinValue(), "AKXcXcIqsqOJFsdwxZ");
    Assert.assertEquals(dimensionColumnMetadata.getMaxValue(), "yQkJTLOQoOqqhkAClgC");
    Assert.assertEquals(metricColumnMetadata.getMinValue(), 890662862);
    Assert.assertEquals(metricColumnMetadata.getMaxValue(), 890662862);
}

Example 10 with IndexLoadingConfigMetadata

use of com.linkedin.pinot.common.metadata.segment.IndexLoadingConfigMetadata in project pinot by linkedin.

the class FilterOperatorBenchmark method main.

public static void main(String[] args) throws Exception {
    String rootDir = args[0];
    File[] segmentDirs = new File(rootDir).listFiles();
    String query = args[1];
    AtomicInteger totalDocsMatched = new AtomicInteger(0);
    Pql2Compiler pql2Compiler = new Pql2Compiler();
    BrokerRequest brokerRequest = pql2Compiler.compileToBrokerRequest(query);
    List<Callable<Void>> segmentProcessors = new ArrayList<>();
    long[] timesSpent = new long[segmentDirs.length];
    for (int i = 0; i < segmentDirs.length; i++) {
        File indexSegmentDir = segmentDirs[i];
        System.out.println("Loading " + indexSegmentDir.getName());
        Configuration tableDataManagerConfig = new PropertiesConfiguration();
        List<String> invertedColumns = new ArrayList<>();
        FilenameFilter filter = new FilenameFilter() {

            @Override
            public boolean accept(File dir, String name) {
                return name.endsWith(".bitmap.inv");
            }
        };
        String[] indexFiles = indexSegmentDir.list(filter);
        for (String indexFileName : indexFiles) {
            invertedColumns.add(indexFileName.replace(".bitmap.inv", ""));
        }
        tableDataManagerConfig.setProperty(IndexLoadingConfigMetadata.KEY_OF_LOADING_INVERTED_INDEX, invertedColumns);
        IndexLoadingConfigMetadata indexLoadingConfigMetadata = new IndexLoadingConfigMetadata(tableDataManagerConfig);
        IndexSegmentImpl indexSegmentImpl = (IndexSegmentImpl) Loaders.IndexSegment.load(indexSegmentDir, ReadMode.heap, indexLoadingConfigMetadata);
        segmentProcessors.add(new SegmentProcessor(i, indexSegmentImpl, brokerRequest, totalDocsMatched, timesSpent));
    }
    ExecutorService executorService = Executors.newCachedThreadPool();
    for (int run = 0; run < 5; run++) {
        System.out.println("START RUN:" + run);
        totalDocsMatched.set(0);
        long start = System.currentTimeMillis();
        List<Future<Void>> futures = executorService.invokeAll(segmentProcessors);
        for (int i = 0; i < futures.size(); i++) {
            futures.get(i).get();
        }
        long end = System.currentTimeMillis();
        System.out.println("Total docs matched:" + totalDocsMatched + " took:" + (end - start));
        System.out.println("Times spent:" + Arrays.toString(timesSpent));
        System.out.println("END RUN:" + run);
    }
    System.exit(0);
}