Search in sources :

Example 1 with RealtimeSegment

use of com.linkedin.pinot.core.realtime.RealtimeSegment in project pinot by linkedin.

the class RealtimeTableDataManagerTest method testSetup.

public void testSetup() throws Exception {
    final HLRealtimeSegmentDataManager manager = new HLRealtimeSegmentDataManager(realtimeSegmentZKMetadata, tableConfig, instanceZKMetadata, null, tableDataManagerConfig.getDataDir(), ReadMode.valueOf(tableDataManagerConfig.getReadMode()), getTestSchema(), new ServerMetrics(new MetricsRegistry()));
    final long start = System.currentTimeMillis();
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            if (System.currentTimeMillis() - start >= (SEGMENT_CONSUMING_TIME)) {
                keepOnRunning = false;
            }
        }
    }, 1000, 1000 * 60 * 1);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("count");
                BlockValSet valSet = mDs.nextBlock().getBlockValueSet();
                BlockSingleValIterator valIt = (BlockSingleValIterator) valSet.iterator();
                int val = valIt.nextIntVal();
                while (val != Constants.EOF) {
                    val = valIt.nextIntVal();
                    sum += val;
                }
            } catch (Exception e) {
                LOGGER.info("count column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan metric col count : " + (stop - start) + " sum : " + sum);
        }
    }, 20000, 1000 * 5);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("viewerId");
                BlockValSet valSet = mDs.nextBlock().getBlockValueSet();
                BlockSingleValIterator valIt = (BlockSingleValIterator) valSet.iterator();
                int val = valIt.nextIntVal();
                while (val != Constants.EOF) {
                    val = valIt.nextIntVal();
                    sum += val;
                }
            } catch (Exception e) {
                LOGGER.info("viewerId column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan SV dimension col viewerId : " + (stop - start) + " sum : " + sum);
        }
    }, 20000, 1000 * 5);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("daysSinceEpoch");
                BlockValSet valSet = mDs.nextBlock().getBlockValueSet();
                BlockSingleValIterator valIt = (BlockSingleValIterator) valSet.iterator();
                int val = valIt.nextIntVal();
                while (val != Constants.EOF) {
                    val = valIt.nextIntVal();
                    sum += val;
                }
            } catch (Exception e) {
                LOGGER.info("daysSinceEpoch column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan SV time col daysSinceEpoch : " + (stop - start) + " sum : " + sum);
        }
    }, 20000, 1000 * 5);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            float sumOfLengths = 0F;
            float counter = 0F;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("viewerCompanies");
                Block b = mDs.nextBlock();
                BlockValSet valSet = b.getBlockValueSet();
                BlockMultiValIterator valIt = (BlockMultiValIterator) valSet.iterator();
                BlockMetadata m = b.getMetadata();
                int maxVams = m.getMaxNumberOfMultiValues();
                while (valIt.hasNext()) {
                    int[] vals = new int[maxVams];
                    int len = valIt.nextIntVal(vals);
                    for (int i = 0; i < len; i++) {
                        sum += vals[i];
                    }
                    sumOfLengths += len;
                    counter++;
                }
            } catch (Exception e) {
                LOGGER.info("daysSinceEpoch column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan MV col viewerCompanies : " + (stop - start) + " sum : " + sum + " average len : " + (sumOfLengths / counter));
        }
    }, 20000, 1000 * 5);
    while (keepOnRunning) {
    // Wait for keepOnRunning to be set to false
    }
}
Also used : MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) IOException(java.io.IOException) FileNotFoundException(java.io.FileNotFoundException) ConfigurationException(org.apache.commons.configuration.ConfigurationException) BlockMultiValIterator(com.linkedin.pinot.core.common.BlockMultiValIterator) TimerTask(java.util.TimerTask) RealtimeColumnDataSource(com.linkedin.pinot.core.realtime.impl.datasource.RealtimeColumnDataSource) BlockSingleValIterator(com.linkedin.pinot.core.common.BlockSingleValIterator) BlockMetadata(com.linkedin.pinot.core.common.BlockMetadata) BlockValSet(com.linkedin.pinot.core.common.BlockValSet) Block(com.linkedin.pinot.core.common.Block) ServerMetrics(com.linkedin.pinot.common.metrics.ServerMetrics) HLRealtimeSegmentDataManager(com.linkedin.pinot.core.data.manager.realtime.HLRealtimeSegmentDataManager) RealtimeSegment(com.linkedin.pinot.core.realtime.RealtimeSegment)

Aggregations

ServerMetrics (com.linkedin.pinot.common.metrics.ServerMetrics)1 Block (com.linkedin.pinot.core.common.Block)1 BlockMetadata (com.linkedin.pinot.core.common.BlockMetadata)1 BlockMultiValIterator (com.linkedin.pinot.core.common.BlockMultiValIterator)1 BlockSingleValIterator (com.linkedin.pinot.core.common.BlockSingleValIterator)1 BlockValSet (com.linkedin.pinot.core.common.BlockValSet)1 HLRealtimeSegmentDataManager (com.linkedin.pinot.core.data.manager.realtime.HLRealtimeSegmentDataManager)1 RealtimeSegment (com.linkedin.pinot.core.realtime.RealtimeSegment)1 RealtimeColumnDataSource (com.linkedin.pinot.core.realtime.impl.datasource.RealtimeColumnDataSource)1 MetricsRegistry (com.yammer.metrics.core.MetricsRegistry)1 FileNotFoundException (java.io.FileNotFoundException)1 IOException (java.io.IOException)1 TimerTask (java.util.TimerTask)1 ConfigurationException (org.apache.commons.configuration.ConfigurationException)1