Examples with BlockValSet com.linkedin.pinot.core.common.BlockValSet used on opensource projects

Example 1 with BlockValSet

use of com.linkedin.pinot.core.common.BlockValSet in project pinot by linkedin.

the class RealtimeTableDataManagerTest method testSetup.

public void testSetup() throws Exception {
    final HLRealtimeSegmentDataManager manager = new HLRealtimeSegmentDataManager(realtimeSegmentZKMetadata, tableConfig, instanceZKMetadata, null, tableDataManagerConfig.getDataDir(), ReadMode.valueOf(tableDataManagerConfig.getReadMode()), getTestSchema(), new ServerMetrics(new MetricsRegistry()));
    final long start = System.currentTimeMillis();
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            if (System.currentTimeMillis() - start >= (SEGMENT_CONSUMING_TIME)) {
                keepOnRunning = false;
            }
        }
    }, 1000, 1000 * 60 * 1);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("count");
                BlockValSet valSet = mDs.nextBlock().getBlockValueSet();
                BlockSingleValIterator valIt = (BlockSingleValIterator) valSet.iterator();
                int val = valIt.nextIntVal();
                while (val != Constants.EOF) {
                    val = valIt.nextIntVal();
                    sum += val;
                }
            } catch (Exception e) {
                LOGGER.info("count column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan metric col count : " + (stop - start) + " sum : " + sum);
        }
    }, 20000, 1000 * 5);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("viewerId");
                BlockValSet valSet = mDs.nextBlock().getBlockValueSet();
                BlockSingleValIterator valIt = (BlockSingleValIterator) valSet.iterator();
                int val = valIt.nextIntVal();
                while (val != Constants.EOF) {
                    val = valIt.nextIntVal();
                    sum += val;
                }
            } catch (Exception e) {
                LOGGER.info("viewerId column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan SV dimension col viewerId : " + (stop - start) + " sum : " + sum);
        }
    }, 20000, 1000 * 5);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("daysSinceEpoch");
                BlockValSet valSet = mDs.nextBlock().getBlockValueSet();
                BlockSingleValIterator valIt = (BlockSingleValIterator) valSet.iterator();
                int val = valIt.nextIntVal();
                while (val != Constants.EOF) {
                    val = valIt.nextIntVal();
                    sum += val;
                }
            } catch (Exception e) {
                LOGGER.info("daysSinceEpoch column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan SV time col daysSinceEpoch : " + (stop - start) + " sum : " + sum);
        }
    }, 20000, 1000 * 5);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            float sumOfLengths = 0F;
            float counter = 0F;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("viewerCompanies");
                Block b = mDs.nextBlock();
                BlockValSet valSet = b.getBlockValueSet();
                BlockMultiValIterator valIt = (BlockMultiValIterator) valSet.iterator();
                BlockMetadata m = b.getMetadata();
                int maxVams = m.getMaxNumberOfMultiValues();
                while (valIt.hasNext()) {
                    int[] vals = new int[maxVams];
                    int len = valIt.nextIntVal(vals);
                    for (int i = 0; i < len; i++) {
                        sum += vals[i];
                    }
                    sumOfLengths += len;
                    counter++;
                }
            } catch (Exception e) {
                LOGGER.info("daysSinceEpoch column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan MV col viewerCompanies : " + (stop - start) + " sum : " + sum + " average len : " + (sumOfLengths / counter));
        }
    }, 20000, 1000 * 5);
    while (keepOnRunning) {
    // Wait for keepOnRunning to be set to false
    }
}

Example 2 with BlockValSet

use of com.linkedin.pinot.core.common.BlockValSet in project pinot by linkedin.

the class DefaultGroupByExecutor method aggregateColumn.

/**
   * Helper method to perform aggregation for a given column.
   *
   * @param transformBlock Transform block to aggregate
   * @param aggrFuncContext Aggregation function context
   * @param resultHolder Holder for results of aggregation
   */
@SuppressWarnings("ConstantConditions")
private void aggregateColumn(TransformBlock transformBlock, AggregationFunctionContext aggrFuncContext, GroupByResultHolder resultHolder) {
    AggregationFunction aggregationFunction = aggrFuncContext.getAggregationFunction();
    String[] aggregationColumns = aggrFuncContext.getAggregationColumns();
    Preconditions.checkState(aggregationColumns.length == 1);
    int length = transformBlock.getNumDocs();
    if (!aggregationFunction.getName().equals(AggregationFunctionFactory.AggregationFunctionType.COUNT.getName())) {
        BlockValSet blockValueSet = transformBlock.getBlockValueSet(aggregationColumns[0]);
        if (_hasMVGroupByColumns) {
            aggregationFunction.aggregateGroupByMV(length, _docIdToMVGroupKey, resultHolder, blockValueSet);
        } else {
            aggregationFunction.aggregateGroupBySV(length, _docIdToSVGroupKey, resultHolder, blockValueSet);
        }
    } else {
        if (_hasMVGroupByColumns) {
            aggregationFunction.aggregateGroupByMV(length, _docIdToMVGroupKey, resultHolder);
        } else {
            aggregationFunction.aggregateGroupBySV(length, _docIdToSVGroupKey, resultHolder);
        }
    }
}

Example 3 with BlockValSet

use of com.linkedin.pinot.core.common.BlockValSet in project pinot by linkedin.

the class NoDictionaryMultiColumnGroupKeyGenerator method generateKeysForBlock.

@Override
public void generateKeysForBlock(TransformBlock transformBlock, int[] docIdToGroupKey) {
    int numGroupByColumns = _groupByColumns.length;
    int numDocs = transformBlock.getNumDocs();
    Object[] values = new Object[numGroupByColumns];
    boolean[] hasDictionary = new boolean[numGroupByColumns];
    FieldSpec.DataType[] dataTypes = new FieldSpec.DataType[numGroupByColumns];
    for (int i = 0; i < numGroupByColumns; i++) {
        BlockValSet blockValSet = transformBlock.getBlockValueSet(_groupByColumns[i]);
        dataTypes[i] = blockValSet.getValueType();
        BlockMetadata blockMetadata = transformBlock.getBlockMetadata(_groupByColumns[i]);
        if (blockMetadata.hasDictionary()) {
            hasDictionary[i] = true;
            values[i] = blockValSet.getDictionaryIds();
        } else {
            hasDictionary[i] = false;
            values[i] = getValuesFromBlockValSet(blockValSet, dataTypes[i]);
        }
    }
    for (int i = 0; i < numDocs; i++) {
        int[] keys = new int[numGroupByColumns];
        for (int j = 0; j < numGroupByColumns; j++) {
            if (hasDictionary[j]) {
                int[] dictIds = (int[]) values[j];
                keys[j] = dictIds[i];
            } else {
                switch(dataTypes[j]) {
                    case INT:
                        int[] intValues = (int[]) values[j];
                        keys[j] = _onTheFlyDictionaries[j].put(intValues[i]);
                        break;
                    case LONG:
                        long[] longValues = (long[]) values[j];
                        keys[j] = _onTheFlyDictionaries[j].put(longValues[i]);
                        break;
                    case FLOAT:
                        float[] floatValues = (float[]) values[j];
                        keys[j] = _onTheFlyDictionaries[j].put(floatValues[i]);
                        break;
                    case DOUBLE:
                        double[] doubleValues = (double[]) values[j];
                        keys[j] = _onTheFlyDictionaries[j].put(doubleValues[i]);
                        break;
                    case STRING:
                        String[] stringValues = (String[]) values[j];
                        keys[j] = _onTheFlyDictionaries[j].put(stringValues[i]);
                        break;
                    default:
                        throw new IllegalArgumentException("Illegal data type for no-dictionary key generator: " + dataTypes[j]);
                }
            }
        }
        docIdToGroupKey[i] = getGroupIdForKey(new FixedIntArray(keys));
    }
}

Example 4 with BlockValSet

use of com.linkedin.pinot.core.common.BlockValSet in project pinot by linkedin.

the class RealtimeSegmentTest method test2.

@Test
public void test2() throws Exception {
    DataSource ds = segmentWithoutInvIdx.getDataSource("column1");
    Block b = ds.nextBlock();
    BlockValSet set = b.getBlockValueSet();
    BlockSingleValIterator it = (BlockSingleValIterator) set.iterator();
    BlockMetadata metadata = b.getMetadata();
    while (it.next()) {
        int dicId = it.nextIntVal();
    }
}

Example 5 with BlockValSet

use of com.linkedin.pinot.core.common.BlockValSet in project pinot by linkedin.

the class ChunkIndexCreationDriverImplTest method test3.

@Test
public void test3() throws Exception {
    final IndexSegmentImpl segment = (IndexSegmentImpl) Loaders.IndexSegment.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    final DataSource ds = segment.getDataSource("column7");
    final Block bl = ds.nextBlock();
    final BlockValSet valSet = bl.getBlockValueSet();
    final int maxValue = ((SegmentMetadataImpl) segment.getSegmentMetadata()).getColumnMetadataFor("column7").getMaxNumberOfMultiValues();
    final BlockMultiValIterator it = (BlockMultiValIterator) valSet.iterator();
    while (it.hasNext()) {
        final int[] entry = new int[maxValue];
        it.nextIntVal(entry);
        LOGGER.trace(Arrays.toString(entry));
    }
}