Examples with BlockMetadata com.linkedin.pinot.core.common.BlockMetadata used on opensource projects

Example 6 with BlockMetadata

use of com.linkedin.pinot.core.common.BlockMetadata in project pinot by linkedin.

the class RealtimeTableDataManagerTest method testSetup.

public void testSetup() throws Exception {
    final HLRealtimeSegmentDataManager manager = new HLRealtimeSegmentDataManager(realtimeSegmentZKMetadata, tableConfig, instanceZKMetadata, null, tableDataManagerConfig.getDataDir(), ReadMode.valueOf(tableDataManagerConfig.getReadMode()), getTestSchema(), new ServerMetrics(new MetricsRegistry()));
    final long start = System.currentTimeMillis();
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            if (System.currentTimeMillis() - start >= (SEGMENT_CONSUMING_TIME)) {
                keepOnRunning = false;
            }
        }
    }, 1000, 1000 * 60 * 1);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("count");
                BlockValSet valSet = mDs.nextBlock().getBlockValueSet();
                BlockSingleValIterator valIt = (BlockSingleValIterator) valSet.iterator();
                int val = valIt.nextIntVal();
                while (val != Constants.EOF) {
                    val = valIt.nextIntVal();
                    sum += val;
                }
            } catch (Exception e) {
                LOGGER.info("count column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan metric col count : " + (stop - start) + " sum : " + sum);
        }
    }, 20000, 1000 * 5);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("viewerId");
                BlockValSet valSet = mDs.nextBlock().getBlockValueSet();
                BlockSingleValIterator valIt = (BlockSingleValIterator) valSet.iterator();
                int val = valIt.nextIntVal();
                while (val != Constants.EOF) {
                    val = valIt.nextIntVal();
                    sum += val;
                }
            } catch (Exception e) {
                LOGGER.info("viewerId column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan SV dimension col viewerId : " + (stop - start) + " sum : " + sum);
        }
    }, 20000, 1000 * 5);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("daysSinceEpoch");
                BlockValSet valSet = mDs.nextBlock().getBlockValueSet();
                BlockSingleValIterator valIt = (BlockSingleValIterator) valSet.iterator();
                int val = valIt.nextIntVal();
                while (val != Constants.EOF) {
                    val = valIt.nextIntVal();
                    sum += val;
                }
            } catch (Exception e) {
                LOGGER.info("daysSinceEpoch column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan SV time col daysSinceEpoch : " + (stop - start) + " sum : " + sum);
        }
    }, 20000, 1000 * 5);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            float sumOfLengths = 0F;
            float counter = 0F;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("viewerCompanies");
                Block b = mDs.nextBlock();
                BlockValSet valSet = b.getBlockValueSet();
                BlockMultiValIterator valIt = (BlockMultiValIterator) valSet.iterator();
                BlockMetadata m = b.getMetadata();
                int maxVams = m.getMaxNumberOfMultiValues();
                while (valIt.hasNext()) {
                    int[] vals = new int[maxVams];
                    int len = valIt.nextIntVal(vals);
                    for (int i = 0; i < len; i++) {
                        sum += vals[i];
                    }
                    sumOfLengths += len;
                    counter++;
                }
            } catch (Exception e) {
                LOGGER.info("daysSinceEpoch column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan MV col viewerCompanies : " + (stop - start) + " sum : " + sum + " average len : " + (sumOfLengths / counter));
        }
    }, 20000, 1000 * 5);
    while (keepOnRunning) {
    // Wait for keepOnRunning to be set to false
    }
}

Example 7 with BlockMetadata

use of com.linkedin.pinot.core.common.BlockMetadata in project pinot by linkedin.

the class DefaultGroupByExecutor method initGroupBy.

/**
   * Initializes the following:
   * <p> - Group key generator. </p>
   * <p> - Result holders </p>
   * <p> - Re-usable storage (eg docId to group key mapping) </p>
   *
   * This is separate from init(), as this can only happen within process as transform block is
   * required to create group key generator.
   *
   * @param transformBlock Transform block to group by.
   */
private void initGroupBy(TransformBlock transformBlock) {
    if (_groupByInited) {
        return;
    }
    FieldSpec.DataType dataType = null;
    for (String groupByColumn : _groupByColumns) {
        BlockMetadata metadata = transformBlock.getBlockMetadata(groupByColumn);
        if (!metadata.isSingleValue()) {
            _hasMVGroupByColumns = true;
        }
        if (!metadata.hasDictionary()) {
            _hasColumnsWithoutDictionary = true;
        }
        // Used only for single group-by case, so ok to overwrite.
        dataType = metadata.getDataType();
    }
    if (_hasColumnsWithoutDictionary) {
        if (_groupByColumns.length == 1) {
            _groupKeyGenerator = new NoDictionarySingleColumnGroupKeyGenerator(_groupByColumns[0], dataType);
        } else {
            _groupKeyGenerator = new NoDictionaryMultiColumnGroupKeyGenerator(transformBlock, _groupByColumns);
        }
    } else {
        _groupKeyGenerator = new DefaultGroupKeyGenerator(transformBlock, _groupByColumns);
    }
    int maxNumResults = _groupKeyGenerator.getGlobalGroupKeyUpperBound();
    initResultHolderArray(_numGroupsLimit, maxNumResults);
    initDocIdToGroupKeyMap();
    _groupByInited = true;
}

Example 8 with BlockMetadata

use of com.linkedin.pinot.core.common.BlockMetadata in project pinot by linkedin.

the class RealtimeFileBasedReaderTest method testDataSourceWithoutPredicateForMultiValueDimensionColumns.

private void testDataSourceWithoutPredicateForMultiValueDimensionColumns() {
    for (FieldSpec spec : schema.getAllFieldSpecs()) {
        if (!spec.isSingleValueField()) {
            DataSource offlineDS = offlineSegment.getDataSource(spec.getName());
            DataSource realtimeDS = realtimeSegment.getDataSource(spec.getName());
            Block offlineBlock = offlineDS.nextBlock();
            Block realtimeBlock = realtimeDS.nextBlock();
            BlockMetadata offlineMetadata = offlineBlock.getMetadata();
            BlockMetadata realtimeMetadata = realtimeBlock.getMetadata();
            BlockMultiValIterator offlineValIterator = (BlockMultiValIterator) offlineBlock.getBlockValueSet().iterator();
            BlockMultiValIterator realtimeValIterator = (BlockMultiValIterator) realtimeBlock.getBlockValueSet().iterator();
            Assert.assertEquals(offlineSegment.getSegmentMetadata().getTotalDocs(), realtimeSegment.getAggregateDocumentCount());
            while (realtimeValIterator.hasNext()) {
                int[] offlineIds = new int[offlineBlock.getMetadata().getMaxNumberOfMultiValues()];
                int[] realtimeIds = new int[realtimeBlock.getMetadata().getMaxNumberOfMultiValues()];
                int Olen = offlineValIterator.nextIntVal(offlineIds);
                int Rlen = realtimeValIterator.nextIntVal(realtimeIds);
                Assert.assertEquals(Olen, Rlen);
                for (int i = 0; i < Olen; i++) {
                    Assert.assertEquals(offlineMetadata.getDictionary().get(offlineIds[i]), realtimeMetadata.getDictionary().get(realtimeIds[i]));
                }
            }
        }
    }
}

Example 9 with BlockMetadata

use of com.linkedin.pinot.core.common.BlockMetadata in project pinot by linkedin.

the class RealtimeSegmentTest method test1.

@Test
public void test1() throws Exception {
    DataSource ds = segmentWithInvIdx.getDataSource("column1");
    Block b = ds.nextBlock();
    BlockValSet set = b.getBlockValueSet();
    BlockSingleValIterator it = (BlockSingleValIterator) set.iterator();
    BlockMetadata metadata = b.getMetadata();
    while (it.next()) {
        int dicId = it.nextIntVal();
    }
}

Example 10 with BlockMetadata

use of com.linkedin.pinot.core.common.BlockMetadata in project pinot by linkedin.

the class ScanBasedFilterOperator method nextFilterBlock.

@Override
public BaseFilterBlock nextFilterBlock(BlockId BlockId) {
    DataSourceMetadata dataSourceMetadata = dataSource.getDataSourceMetadata();
    FilterBlockDocIdSet docIdSet;
    Block nextBlock = dataSource.nextBlock();
    BlockValSet blockValueSet = nextBlock.getBlockValueSet();
    BlockMetadata blockMetadata = nextBlock.getMetadata();
    if (dataSourceMetadata.isSingleValue()) {
        docIdSet = new ScanBasedSingleValueDocIdSet(dataSource.getOperatorName(), blockValueSet, blockMetadata, predicateEvaluator);
    } else {
        docIdSet = new ScanBasedMultiValueDocIdSet(dataSource.getOperatorName(), blockValueSet, blockMetadata, predicateEvaluator);
    }
    if (startDocId != null) {
        docIdSet.setStartDocId(startDocId);
    }
    if (endDocId != null) {
        docIdSet.setEndDocId(endDocId);
    }
    return new ScanBlock(docIdSet);
}