Examples with IndexSegment com.linkedin.pinot.core.indexsegment.IndexSegment used on opensource projects

Example 41 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class TablesResourceTest method getTables.

@Test
public void getTables() throws Exception {
    Response response = testHelper.target.path("/tables").request().get(Response.class);
    String responseBody = response.readEntity(String.class);
    TablesList tablesList = new ObjectMapper().readValue(responseBody, TablesList.class);
    assertNotNull(tablesList);
    List<String> tables = tablesList.getTables();
    assertNotNull(tables);
    assertEquals(tables.size(), 1);
    assertEquals(tables.get(0), ResourceTestHelper.DEFAULT_TABLE_NAME);
    final String secondTable = "secondTable";
    testHelper.addTable(secondTable);
    IndexSegment secondSegment = testHelper.setupSegment(secondTable, ResourceTestHelper.DEFAULT_AVRO_DATA_FILE, "2");
    tablesList = testHelper.target.path("/tables").request().get(TablesList.class);
    assertNotNull(tablesList);
    assertNotNull(tablesList.getTables());
    assertEquals(tablesList.getTables().size(), 2);
    assertTrue(tablesList.getTables().contains(ResourceTestHelper.DEFAULT_TABLE_NAME));
    assertTrue(tablesList.getTables().contains(secondTable));
}

Example 42 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class SegmentDumpTool method doMain.

public void doMain(String[] args) throws Exception {
    CmdLineParser parser = new CmdLineParser(this);
    parser.parseArgument(args);
    File segmentDir = new File(segmentPath);
    SegmentMetadata metadata = new SegmentMetadataImpl(segmentDir);
    // All columns by default
    if (columnNames == null) {
        columnNames = new ArrayList<String>(metadata.getSchema().getColumnNames());
        Collections.sort(columnNames);
    }
    IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDir, ReadMode.mmap);
    Map<String, Dictionary> dictionaries = new HashMap<String, Dictionary>();
    Map<String, BlockSingleValIterator> iterators = new HashMap<String, BlockSingleValIterator>();
    for (String columnName : columnNames) {
        DataSource dataSource = indexSegment.getDataSource(columnName);
        dataSource.open();
        Block block = dataSource.nextBlock();
        BlockValSet blockValSet = block.getBlockValueSet();
        BlockSingleValIterator itr = (BlockSingleValIterator) blockValSet.iterator();
        iterators.put(columnName, itr);
        dictionaries.put(columnName, dataSource.getDictionary());
    }
    System.out.print("Doc\t");
    for (String columnName : columnNames) {
        System.out.print(columnName);
        System.out.print("\t");
    }
    System.out.println();
    for (int i = 0; i < indexSegment.getSegmentMetadata().getTotalDocs(); i++) {
        System.out.print(i);
        System.out.print("\t");
        for (String columnName : columnNames) {
            FieldSpec.DataType columnType = metadata.getSchema().getFieldSpecFor(columnName).getDataType();
            BlockSingleValIterator itr = iterators.get(columnName);
            Integer encodedValue = itr.nextIntVal();
            Object value = dictionaries.get(columnName).get(encodedValue);
            System.out.print(value);
            System.out.print("\t");
        }
        System.out.println();
    }
    if (dumpStarTree) {
        System.out.println();
        File starTreeFile = new File(segmentDir, V1Constants.STAR_TREE_INDEX_FILE);
        StarTreeInterf tree = StarTreeSerDe.fromFile(starTreeFile, ReadMode.mmap);
        tree.printTree();
    }
}

Example 43 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class RealtimeTableDataManager method addSegment.

/*
   * This call comes in one of two ways:
   * For HL Segments:
   * - We are being directed by helix to own up all the segments that we committed and are still in retention. In this case
   *   we treat it exactly like how OfflineTableDataManager would -- wrap it into an OfflineSegmentDataManager, and put it
   *   in the map.
   * - We are being asked to own up a new realtime segment. In this case, we wrap the segment with a RealTimeSegmentDataManager
   *   (that kicks off Kafka consumption). When the segment is committed we get notified via the notifySegmentCommitted call, at
   *   which time we replace the segment with the OfflineSegmentDataManager
   * For LL Segments:
   * - We are being asked to start consuming from a kafka partition.
   * - We did not know about the segment and are being asked to download and own the segment (re-balancing, or
   *   replacing a realtime server with a fresh one, maybe). We need to look at segment metadata and decide whether
   *   to start consuming or download the segment.
   */
@Override
public void addSegment(ZkHelixPropertyStore<ZNRecord> propertyStore, AbstractTableConfig tableConfig, InstanceZKMetadata instanceZKMetadata, SegmentZKMetadata inputSegmentZKMetadata) throws Exception {
    // TODO FIXME
    // Hack. We get the _helixPropertyStore here and save it, knowing that we will get this addSegment call
    // before the notifyCommitted call (that uses _helixPropertyStore)
    this._helixPropertyStore = propertyStore;
    final String segmentId = inputSegmentZKMetadata.getSegmentName();
    final String tableName = inputSegmentZKMetadata.getTableName();
    if (!(inputSegmentZKMetadata instanceof RealtimeSegmentZKMetadata)) {
        LOGGER.warn("Got called with an unexpected instance object:{},table {}, segment {}", inputSegmentZKMetadata.getClass().getSimpleName(), tableName, segmentId);
        return;
    }
    RealtimeSegmentZKMetadata segmentZKMetadata = (RealtimeSegmentZKMetadata) inputSegmentZKMetadata;
    LOGGER.info("Attempting to add realtime segment {} for table {}", segmentId, tableName);
    if (new File(_indexDir, segmentId).exists() && (segmentZKMetadata).getStatus() == Status.DONE) {
        // segment already exists on file, and we have committed the realtime segment in ZK. Treat it like an offline segment
        if (_segmentsMap.containsKey(segmentId)) {
            LOGGER.warn("Got reload for segment already on disk {} table {}, have {}", segmentId, tableName, _segmentsMap.get(segmentId).getClass().getSimpleName());
            return;
        }
        IndexSegment segment = ColumnarSegmentLoader.load(new File(_indexDir, segmentId), _readMode, _indexLoadingConfigMetadata);
        addSegment(segment);
        markSegmentAsLoaded(segmentId);
    } else {
        // on-disk segment next time
        if (_segmentsMap.containsKey(segmentId)) {
            LOGGER.warn("Got reload for segment not on disk {} table {}, have {}", segmentId, tableName, _segmentsMap.get(segmentId).getClass().getSimpleName());
            return;
        }
        PinotHelixPropertyStoreZnRecordProvider propertyStoreHelper = PinotHelixPropertyStoreZnRecordProvider.forSchema(propertyStore);
        ZNRecord record = propertyStoreHelper.get(tableConfig.getValidationConfig().getSchemaName());
        LOGGER.info("Found schema {} ", tableConfig.getValidationConfig().getSchemaName());
        Schema schema = SchemaUtils.fromZNRecord(record);
        if (!isValid(schema, tableConfig.getIndexingConfig())) {
            LOGGER.error("Not adding segment {}", segmentId);
            throw new RuntimeException("Mismatching schema/table config for " + _tableName);
        }
        SegmentDataManager manager;
        if (SegmentName.isHighLevelConsumerSegmentName(segmentId)) {
            manager = new HLRealtimeSegmentDataManager(segmentZKMetadata, tableConfig, instanceZKMetadata, this, _indexDir.getAbsolutePath(), _readMode, SchemaUtils.fromZNRecord(record), _serverMetrics);
        } else {
            LLCRealtimeSegmentZKMetadata llcSegmentMetadata = (LLCRealtimeSegmentZKMetadata) segmentZKMetadata;
            if (segmentZKMetadata.getStatus().equals(Status.DONE)) {
                // TODO Remove code duplication here and in LLRealtimeSegmentDataManager
                downloadAndReplaceSegment(segmentId, llcSegmentMetadata);
                return;
            }
            manager = new LLRealtimeSegmentDataManager(segmentZKMetadata, tableConfig, instanceZKMetadata, this, _indexDir.getAbsolutePath(), SchemaUtils.fromZNRecord(record), _serverMetrics);
        }
        LOGGER.info("Initialize RealtimeSegmentDataManager - " + segmentId);
        try {
            _rwLock.writeLock().lock();
            _segmentsMap.put(segmentId, manager);
        } finally {
            _rwLock.writeLock().unlock();
        }
        _loadingSegments.add(segmentId);
    }
}

Example 44 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class OfflineTableDataManager method addSegment.

@Override
public void addSegment(SegmentMetadata segmentMetadata, Schema schema) throws Exception {
    IndexSegment indexSegment = ColumnarSegmentLoader.loadSegment(new File(segmentMetadata.getIndexDir()), _readMode, _indexLoadingConfigMetadata, schema);
    addSegment(indexSegment);
}

Example 45 with IndexSegment

use of com.linkedin.pinot.core.indexsegment.IndexSegment in project pinot by linkedin.

the class SegmentV1V2ToV3FormatConverterTest method testConvert.

@Test
public void testConvert() throws Exception {
    SegmentMetadataImpl beforeConversionMeta = new SegmentMetadataImpl(segmentDirectory);
    SegmentV1V2ToV3FormatConverter converter = new SegmentV1V2ToV3FormatConverter();
    converter.convert(segmentDirectory);
    File v3Location = SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3);
    Assert.assertTrue(v3Location.exists());
    Assert.assertTrue(v3Location.isDirectory());
    Assert.assertTrue(new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).exists());
    SegmentMetadataImpl metadata = new SegmentMetadataImpl(v3Location);
    Assert.assertEquals(metadata.getVersion(), SegmentVersion.v3.toString());
    Assert.assertTrue(new File(v3Location, V1Constants.SEGMENT_CREATION_META).exists());
    // Drop the star tree index file because it has invalid data
    new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).delete();
    new File(segmentDirectory, V1Constants.STAR_TREE_INDEX_FILE).delete();
    FileTime afterConversionTime = Files.getLastModifiedTime(v3Location.toPath());
    // verify that the segment loads correctly. This is necessary and sufficient
    // full proof way to ensure that segment is correctly translated
    IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap, v3LoadingConfig);
    Assert.assertNotNull(indexSegment);
    Assert.assertEquals(indexSegment.getSegmentName(), metadata.getName());
    Assert.assertEquals(SegmentVersion.v3, SegmentVersion.valueOf(indexSegment.getSegmentMetadata().getVersion()));
    FileTime afterLoadTime = Files.getLastModifiedTime(v3Location.toPath());
    Assert.assertEquals(afterConversionTime, afterLoadTime);
    // verify that SegmentMetadataImpl loaded from segmentDirectory correctly sets
    // metadata information after conversion. This has impacted us while loading
    // segments by triggering download. That's costly. That's also difficult to test
    Assert.assertFalse(new File(segmentDirectory, V1Constants.MetadataKeys.METADATA_FILE_NAME).exists());
    SegmentMetadataImpl metaAfterConversion = new SegmentMetadataImpl(segmentDirectory);
    Assert.assertNotNull(metaAfterConversion);
    Assert.assertFalse(metaAfterConversion.getCrc().equalsIgnoreCase(String.valueOf(Long.MIN_VALUE)));
    Assert.assertEquals(metaAfterConversion.getCrc(), beforeConversionMeta.getCrc());
    Assert.assertTrue(metaAfterConversion.getIndexCreationTime() != Long.MIN_VALUE);
    Assert.assertEquals(metaAfterConversion.getIndexCreationTime(), beforeConversionMeta.getIndexCreationTime());
}