Examples with Block com.linkedin.pinot.core.common.Block used on opensource projects

Example 1 with Block

use of com.linkedin.pinot.core.common.Block in project pinot by linkedin.

the class RealtimeTableDataManagerTest method testSetup.

public void testSetup() throws Exception {
    final HLRealtimeSegmentDataManager manager = new HLRealtimeSegmentDataManager(realtimeSegmentZKMetadata, tableConfig, instanceZKMetadata, null, tableDataManagerConfig.getDataDir(), ReadMode.valueOf(tableDataManagerConfig.getReadMode()), getTestSchema(), new ServerMetrics(new MetricsRegistry()));
    final long start = System.currentTimeMillis();
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            if (System.currentTimeMillis() - start >= (SEGMENT_CONSUMING_TIME)) {
                keepOnRunning = false;
            }
        }
    }, 1000, 1000 * 60 * 1);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("count");
                BlockValSet valSet = mDs.nextBlock().getBlockValueSet();
                BlockSingleValIterator valIt = (BlockSingleValIterator) valSet.iterator();
                int val = valIt.nextIntVal();
                while (val != Constants.EOF) {
                    val = valIt.nextIntVal();
                    sum += val;
                }
            } catch (Exception e) {
                LOGGER.info("count column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan metric col count : " + (stop - start) + " sum : " + sum);
        }
    }, 20000, 1000 * 5);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("viewerId");
                BlockValSet valSet = mDs.nextBlock().getBlockValueSet();
                BlockSingleValIterator valIt = (BlockSingleValIterator) valSet.iterator();
                int val = valIt.nextIntVal();
                while (val != Constants.EOF) {
                    val = valIt.nextIntVal();
                    sum += val;
                }
            } catch (Exception e) {
                LOGGER.info("viewerId column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan SV dimension col viewerId : " + (stop - start) + " sum : " + sum);
        }
    }, 20000, 1000 * 5);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("daysSinceEpoch");
                BlockValSet valSet = mDs.nextBlock().getBlockValueSet();
                BlockSingleValIterator valIt = (BlockSingleValIterator) valSet.iterator();
                int val = valIt.nextIntVal();
                while (val != Constants.EOF) {
                    val = valIt.nextIntVal();
                    sum += val;
                }
            } catch (Exception e) {
                LOGGER.info("daysSinceEpoch column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan SV time col daysSinceEpoch : " + (stop - start) + " sum : " + sum);
        }
    }, 20000, 1000 * 5);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            float sumOfLengths = 0F;
            float counter = 0F;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("viewerCompanies");
                Block b = mDs.nextBlock();
                BlockValSet valSet = b.getBlockValueSet();
                BlockMultiValIterator valIt = (BlockMultiValIterator) valSet.iterator();
                BlockMetadata m = b.getMetadata();
                int maxVams = m.getMaxNumberOfMultiValues();
                while (valIt.hasNext()) {
                    int[] vals = new int[maxVams];
                    int len = valIt.nextIntVal(vals);
                    for (int i = 0; i < len; i++) {
                        sum += vals[i];
                    }
                    sumOfLengths += len;
                    counter++;
                }
            } catch (Exception e) {
                LOGGER.info("daysSinceEpoch column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan MV col viewerCompanies : " + (stop - start) + " sum : " + sum + " average len : " + (sumOfLengths / counter));
        }
    }, 20000, 1000 * 5);
    while (keepOnRunning) {
    // Wait for keepOnRunning to be set to false
    }
}

Example 2 with Block

use of com.linkedin.pinot.core.common.Block in project pinot by linkedin.

the class MCombineOperator method getNextBlock.

@Override
public Block getNextBlock() {
    final long startTime = System.currentTimeMillis();
    final long queryEndTime = System.currentTimeMillis() + _timeOutMs;
    int numGroups = Math.max(MIN_THREADS_PER_QUERY, Math.min(MAX_THREADS_PER_QUERY, (_operators.size() + MIN_SEGMENTS_PER_THREAD - 1) / MIN_SEGMENTS_PER_THREAD));
    //ensure that the number of groups is not more than the number of segments
    numGroups = Math.min(_operators.size(), numGroups);
    final List<List<Operator>> operatorGroups = new ArrayList<List<Operator>>(numGroups);
    for (int i = 0; i < numGroups; i++) {
        operatorGroups.add(new ArrayList<Operator>());
    }
    for (int i = 0; i < _operators.size(); i++) {
        operatorGroups.get(i % numGroups).add(_operators.get(i));
    }
    final BlockingQueue<Block> blockingQueue = new ArrayBlockingQueue<>(operatorGroups.size());
    // Submit operators.
    for (final List<Operator> operatorGroup : operatorGroups) {
        _executorService.submit(new TraceRunnable() {

            @Override
            public void runJob() {
                IntermediateResultsBlock mergedBlock = null;
                try {
                    for (Operator operator : operatorGroup) {
                        IntermediateResultsBlock blockToMerge = (IntermediateResultsBlock) operator.nextBlock();
                        if (mergedBlock == null) {
                            mergedBlock = blockToMerge;
                        } else {
                            try {
                                CombineService.mergeTwoBlocks(_brokerRequest, mergedBlock, blockToMerge);
                            } catch (Exception e) {
                                LOGGER.error("Caught exception while merging two blocks (step 1).", e);
                                mergedBlock.addToProcessingExceptions(QueryException.getException(QueryException.MERGE_RESPONSE_ERROR, e));
                            }
                        }
                    }
                } catch (Exception e) {
                    LOGGER.error("Caught exception while executing query.", e);
                    mergedBlock = new IntermediateResultsBlock(e);
                }
                blockingQueue.offer(mergedBlock);
            }
        });
    }
    LOGGER.debug("Submitting operators to be run in parallel and it took:" + (System.currentTimeMillis() - startTime));
    // Submit merger job:
    Future<IntermediateResultsBlock> mergedBlockFuture = _executorService.submit(new TraceCallable<IntermediateResultsBlock>() {

        @Override
        public IntermediateResultsBlock callJob() throws Exception {
            int mergedBlocksNumber = 0;
            IntermediateResultsBlock mergedBlock = null;
            while ((queryEndTime > System.currentTimeMillis()) && (mergedBlocksNumber < operatorGroups.size())) {
                if (mergedBlock == null) {
                    mergedBlock = (IntermediateResultsBlock) blockingQueue.poll(queryEndTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS);
                    if (mergedBlock != null) {
                        mergedBlocksNumber++;
                    }
                    LOGGER.debug("Got response from operator 0 after: {}", (System.currentTimeMillis() - startTime));
                } else {
                    IntermediateResultsBlock blockToMerge = (IntermediateResultsBlock) blockingQueue.poll(queryEndTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS);
                    if (blockToMerge != null) {
                        try {
                            LOGGER.debug("Got response from operator {} after: {}", mergedBlocksNumber, (System.currentTimeMillis() - startTime));
                            CombineService.mergeTwoBlocks(_brokerRequest, mergedBlock, blockToMerge);
                            LOGGER.debug("Merged response from operator {} after: {}", mergedBlocksNumber, (System.currentTimeMillis() - startTime));
                        } catch (Exception e) {
                            LOGGER.error("Caught exception while merging two blocks (step 2).", e);
                            mergedBlock.addToProcessingExceptions(QueryException.getException(QueryException.MERGE_RESPONSE_ERROR, e));
                        }
                        mergedBlocksNumber++;
                    }
                }
            }
            return mergedBlock;
        }
    });
    // Get merge results.
    IntermediateResultsBlock mergedBlock;
    try {
        mergedBlock = mergedBlockFuture.get(queryEndTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
        LOGGER.error("Caught InterruptedException.", e);
        mergedBlock = new IntermediateResultsBlock(QueryException.getException(QueryException.FUTURE_CALL_ERROR, e));
    } catch (ExecutionException e) {
        LOGGER.error("Caught ExecutionException.", e);
        mergedBlock = new IntermediateResultsBlock(QueryException.getException(QueryException.MERGE_RESPONSE_ERROR, e));
    } catch (TimeoutException e) {
        LOGGER.error("Caught TimeoutException", e);
        mergedBlock = new IntermediateResultsBlock(QueryException.getException(QueryException.EXECUTION_TIMEOUT_ERROR, e));
    }
    // Update execution statistics.
    ExecutionStatistics executionStatistics = new ExecutionStatistics();
    for (Operator operator : _operators) {
        ExecutionStatistics executionStatisticsToMerge = operator.getExecutionStatistics();
        if (executionStatisticsToMerge != null) {
            executionStatistics.merge(executionStatisticsToMerge);
        }
    }
    mergedBlock.setNumDocsScanned(executionStatistics.getNumDocsScanned());
    mergedBlock.setNumEntriesScannedInFilter(executionStatistics.getNumEntriesScannedInFilter());
    mergedBlock.setNumEntriesScannedPostFilter(executionStatistics.getNumEntriesScannedPostFilter());
    mergedBlock.setNumTotalRawDocs(executionStatistics.getNumTotalRawDocs());
    return mergedBlock;
}

Example 3 with Block

use of com.linkedin.pinot.core.common.Block in project pinot by linkedin.

the class OrOperator method nextFilterBlock.

@Override
public BaseFilterBlock nextFilterBlock(BlockId BlockId) {
    List<FilterBlockDocIdSet> blockDocIdSets = new ArrayList<FilterBlockDocIdSet>();
    for (Operator operator : operators) {
        Block block = operator.nextBlock();
        FilterBlockDocIdSet blockDocIdSet = (FilterBlockDocIdSet) block.getBlockDocIdSet();
        blockDocIdSets.add(blockDocIdSet);
    }
    orBlock = new OrBlock(blockDocIdSets);
    return orBlock;
}

Example 4 with Block

use of com.linkedin.pinot.core.common.Block in project pinot by linkedin.

the class RealtimeColumnDataSource method getBlock.

private Block getBlock() {
    if (!blockReturned) {
        blockReturned = true;
        isPredicateEvaluated = true;
        if (fieldSpec.isSingleValueField()) {
            Block SvBlock = new RealtimeSingleValueBlock(filteredDocIdBitmap, fieldSpec, dictionary, offset, (FixedByteSingleColumnSingleValueReaderWriter) indexReader);
            return SvBlock;
        } else {
            Block mvBlock = new RealtimeMultiValueBlock(fieldSpec, dictionary, filteredDocIdBitmap, offset, maxNumberOfMultiValues, (FixedByteSingleColumnMultiValueReaderWriter) indexReader);
            return mvBlock;
        }
    }
    return null;
}

Example 5 with Block

use of com.linkedin.pinot.core.common.Block in project pinot by linkedin.

the class RealtimeFileBasedReaderTest method testDataSourceWithoutPredicateForSingleValueTimeColumns.

private void testDataSourceWithoutPredicateForSingleValueTimeColumns() {
    for (FieldSpec spec : schema.getAllFieldSpecs()) {
        if (spec.isSingleValueField() && spec.getFieldType() == FieldType.TIME) {
            DataSource offlineDS = offlineSegment.getDataSource(spec.getName());
            DataSource realtimeDS = realtimeSegment.getDataSource(spec.getName());
            Block offlineBlock = offlineDS.nextBlock();
            Block realtimeBlock = realtimeDS.nextBlock();
            BlockMetadata offlineMetadata = offlineBlock.getMetadata();
            BlockMetadata realtimeMetadata = realtimeBlock.getMetadata();
            BlockSingleValIterator offlineValIterator = (BlockSingleValIterator) offlineBlock.getBlockValueSet().iterator();
            BlockSingleValIterator realtimeValIterator = (BlockSingleValIterator) realtimeBlock.getBlockValueSet().iterator();
            Assert.assertEquals(offlineSegment.getSegmentMetadata().getTotalDocs(), realtimeSegment.getAggregateDocumentCount());
            while (realtimeValIterator.hasNext()) {
                int offlineDicId = offlineValIterator.nextIntVal();
                int realtimeDicId = realtimeValIterator.nextIntVal();
                Assert.assertEquals(offlineMetadata.getDictionary().get(offlineDicId), realtimeMetadata.getDictionary().get(realtimeDicId));
            }
            Assert.assertEquals(offlineValIterator.hasNext(), realtimeValIterator.hasNext());
        }
    }
}