Examples with TableBlockInfo org.apache.carbondata.core.datastore.block.TableBlockInfo used on opensource projects

Example 1 with TableBlockInfo

use of org.apache.carbondata.core.datastore.block.TableBlockInfo in project carbondata by apache.

the class SegmentTaskIndexStore method loadAndGetTaskIdToSegmentsMap.

/**
   * Below method will be used to load the segment of segments
   * One segment may have multiple task , so  table segment will be loaded
   * based on task id and will return the map of taksId to table segment
   * map
   *
   * @param segmentToTableBlocksInfos segment id to block info
   * @param absoluteTableIdentifier   absolute table identifier
   * @return map of taks id to segment mapping
   * @throws IOException
   */
private SegmentTaskIndexWrapper loadAndGetTaskIdToSegmentsMap(Map<String, List<TableBlockInfo>> segmentToTableBlocksInfos, AbsoluteTableIdentifier absoluteTableIdentifier, TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier) throws IOException {
    // task id to segment map
    Iterator<Map.Entry<String, List<TableBlockInfo>>> iteratorOverSegmentBlocksInfos = segmentToTableBlocksInfos.entrySet().iterator();
    Map<TaskBucketHolder, AbstractIndex> taskIdToSegmentIndexMap = null;
    SegmentTaskIndexWrapper segmentTaskIndexWrapper = null;
    SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(absoluteTableIdentifier);
    String segmentId = null;
    TaskBucketHolder taskBucketHolder = null;
    try {
        while (iteratorOverSegmentBlocksInfos.hasNext()) {
            // segment id to table block mapping
            Map.Entry<String, List<TableBlockInfo>> next = iteratorOverSegmentBlocksInfos.next();
            // group task id to table block info mapping for the segment
            Map<TaskBucketHolder, List<TableBlockInfo>> taskIdToTableBlockInfoMap = mappedAndGetTaskIdToTableBlockInfo(segmentToTableBlocksInfos);
            segmentId = next.getKey();
            // get the existing map of task id to table segment map
            UpdateVO updateVO = updateStatusManager.getInvalidTimestampRange(segmentId);
            // check if segment is already loaded, if segment is already loaded
            //no need to load the segment block
            String lruCacheKey = tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier();
            segmentTaskIndexWrapper = (SegmentTaskIndexWrapper) lruCache.get(lruCacheKey);
            if (segmentTaskIndexWrapper == null || tableSegmentUniqueIdentifier.isSegmentUpdated()) {
                // get the segment loader lock object this is to avoid
                // same segment is getting loaded multiple times
                // in case of concurrent query
                Object segmentLoderLockObject = segmentLockMap.get(lruCacheKey);
                if (null == segmentLoderLockObject) {
                    segmentLoderLockObject = addAndGetSegmentLock(lruCacheKey);
                }
                // acquire lock to lod the segment
                synchronized (segmentLoderLockObject) {
                    segmentTaskIndexWrapper = (SegmentTaskIndexWrapper) lruCache.get(lruCacheKey);
                    if (null == segmentTaskIndexWrapper || tableSegmentUniqueIdentifier.isSegmentUpdated()) {
                        // so that the same can be updated after loading the btree.
                        if (tableSegmentUniqueIdentifier.isSegmentUpdated() && null != segmentTaskIndexWrapper) {
                            taskIdToSegmentIndexMap = segmentTaskIndexWrapper.getTaskIdToTableSegmentMap();
                        } else {
                            // creating a map of take if to table segment
                            taskIdToSegmentIndexMap = new HashMap<TaskBucketHolder, AbstractIndex>();
                            segmentTaskIndexWrapper = new SegmentTaskIndexWrapper(taskIdToSegmentIndexMap);
                            segmentTaskIndexWrapper.incrementAccessCount();
                        }
                        Iterator<Map.Entry<TaskBucketHolder, List<TableBlockInfo>>> iterator = taskIdToTableBlockInfoMap.entrySet().iterator();
                        long requiredSize = calculateRequiredSize(taskIdToTableBlockInfoMap, absoluteTableIdentifier);
                        segmentTaskIndexWrapper.setMemorySize(requiredSize + segmentTaskIndexWrapper.getMemorySize());
                        boolean isAddedToLruCache = lruCache.put(lruCacheKey, segmentTaskIndexWrapper, requiredSize);
                        if (isAddedToLruCache) {
                            while (iterator.hasNext()) {
                                Map.Entry<TaskBucketHolder, List<TableBlockInfo>> taskToBlockInfoList = iterator.next();
                                taskBucketHolder = taskToBlockInfoList.getKey();
                                taskIdToSegmentIndexMap.put(taskBucketHolder, loadBlocks(taskBucketHolder, taskToBlockInfoList.getValue(), absoluteTableIdentifier));
                            }
                        } else {
                            throw new IndexBuilderException("Can not load the segment. No Enough space available.");
                        }
                        // set the latest timestamp.
                        segmentTaskIndexWrapper.setRefreshedTimeStamp(updateVO.getCreatedOrUpdatedTimeStamp());
                        // tableSegmentMapTemp.put(next.getKey(), taskIdToSegmentIndexMap);
                        // removing from segment lock map as once segment is loaded
                        // if concurrent query is coming for same segment
                        // it will wait on the lock so after this segment will be already
                        // loaded so lock is not required, that is why removing the
                        // the lock object as it wont be useful
                        segmentLockMap.remove(lruCacheKey);
                    } else {
                        segmentTaskIndexWrapper.incrementAccessCount();
                    }
                }
            } else {
                segmentTaskIndexWrapper.incrementAccessCount();
            }
        }
    } catch (IndexBuilderException e) {
        LOGGER.error("Problem while loading the segment");
        throw e;
    }
    return segmentTaskIndexWrapper;
}

Example 2 with TableBlockInfo

use of org.apache.carbondata.core.datastore.block.TableBlockInfo in project carbondata by apache.

the class BlockIndexStore method get.

/**
   * The method loads the block meta in B-tree lru cache and returns the block meta.
   *
   * @param tableBlockUniqueIdentifier Uniquely identifies the block
   * @return returns the blocks B-Tree meta
   */
@Override
public AbstractIndex get(TableBlockUniqueIdentifier tableBlockUniqueIdentifier) throws IOException {
    TableBlockInfo tableBlockInfo = tableBlockUniqueIdentifier.getTableBlockInfo();
    BlockInfo blockInfo = new BlockInfo(tableBlockInfo);
    String lruCacheKey = getLruCacheKey(tableBlockUniqueIdentifier.getAbsoluteTableIdentifier(), blockInfo);
    AbstractIndex tableBlock = (AbstractIndex) lruCache.get(lruCacheKey);
    // if block is not loaded
    if (null == tableBlock) {
        // check any lock object is present in
        // block info lock map
        Object blockInfoLockObject = blockInfoLock.get(blockInfo);
        // object will be added
        if (null == blockInfoLockObject) {
            synchronized (blockInfoLock) {
                // again checking the block info lock, to check whether lock object is present
                // or not if now also not present then add a lock object
                blockInfoLockObject = blockInfoLock.get(blockInfo);
                if (null == blockInfoLockObject) {
                    blockInfoLockObject = new Object();
                    blockInfoLock.put(blockInfo, blockInfoLockObject);
                }
            }
        }
        //acquire the lock for particular block info
        synchronized (blockInfoLockObject) {
            // check again whether block is present or not to avoid the
            // same block is loaded
            //more than once in case of concurrent query
            tableBlock = (AbstractIndex) lruCache.get(getLruCacheKey(tableBlockUniqueIdentifier.getAbsoluteTableIdentifier(), blockInfo));
            // if still block is not present then load the block
            if (null == tableBlock) {
                tableBlock = loadBlock(tableBlockUniqueIdentifier);
                fillSegmentIdToBlockListMap(tableBlockUniqueIdentifier.getAbsoluteTableIdentifier(), blockInfo);
            }
        }
    } else {
        tableBlock.incrementAccessCount();
    }
    return tableBlock;
}

Example 3 with TableBlockInfo

use of org.apache.carbondata.core.datastore.block.TableBlockInfo in project carbondata by apache.

the class CarbonUtilTest method testToReadMetadatFile.

@Test
public void testToReadMetadatFile() throws IOException {
    new MockUp<DataFileFooterConverter>() {

        @SuppressWarnings("unused")
        @Mock
        public DataFileFooter readDataFileFooter(TableBlockInfo info) {
            DataFileFooter fileFooter = new DataFileFooter();
            fileFooter.setVersionId(ColumnarFormatVersion.V1);
            return fileFooter;
        }
    };
    TableBlockInfo info = new TableBlockInfo("file:/", 1, "0", new String[0], 1, ColumnarFormatVersion.V1);
    assertEquals(CarbonUtil.readMetadatFile(info).getVersionId().number(), 1);
}

Example 4 with TableBlockInfo

use of org.apache.carbondata.core.datastore.block.TableBlockInfo in project carbondata by apache.

the class BlockIndexStoreTest method testLoadAndGetTaskIdToSegmentsMapForSingleSegment.

@Test
public void testLoadAndGetTaskIdToSegmentsMapForSingleSegment() throws IOException {
    File file = getPartFile();
    TableBlockInfo info = new TableBlockInfo(file.getAbsolutePath(), 0, "0", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "t3", "1");
    AbsoluteTableIdentifier absoluteTableIdentifier = new AbsoluteTableIdentifier("/src/test/resources", carbonTableIdentifier);
    try {
        List<TableBlockUniqueIdentifier> tableBlockInfoList = getTableBlockUniqueIdentifierList(Arrays.asList(new TableBlockInfo[] { info }), absoluteTableIdentifier);
        List<AbstractIndex> loadAndGetBlocks = cache.getAll(tableBlockInfoList);
        assertTrue(loadAndGetBlocks.size() == 1);
    } catch (Exception e) {
        assertTrue(false);
    }
    List<String> segmentIds = new ArrayList<>();
    segmentIds.add(info.getSegmentId());
    cache.removeTableBlocks(segmentIds, absoluteTableIdentifier);
}

Example 5 with TableBlockInfo

use of org.apache.carbondata.core.datastore.block.TableBlockInfo in project carbondata by apache.

the class DataFileFooterConverterTest method testReadDataFileFooter.

@Test
public void testReadDataFileFooter() throws Exception {
    DataFileFooterConverter dataFileFooterConverter = new DataFileFooterConverter();
    DataFileFooter dataFileFooter = new DataFileFooter();
    List<Integer> column_cardinalities = new ArrayList<>();
    column_cardinalities.add(new Integer("1"));
    column_cardinalities.add(new Integer("2"));
    column_cardinalities.add(new Integer("3"));
    org.apache.carbondata.format.SegmentInfo segmentInfo1 = new org.apache.carbondata.format.SegmentInfo(3, column_cardinalities);
    List<Encoding> encoders = new ArrayList<>();
    encoders.add(Encoding.INVERTED_INDEX);
    encoders.add(Encoding.BIT_PACKED);
    encoders.add(Encoding.DELTA);
    encoders.add(Encoding.DICTIONARY);
    encoders.add(Encoding.DIRECT_DICTIONARY);
    encoders.add(Encoding.RLE);
    ColumnSchema columnSchema = new ColumnSchema(DataType.INT, "column", "3", true, encoders, true);
    ColumnSchema columnSchema1 = new ColumnSchema(DataType.ARRAY, "column", "3", true, encoders, true);
    ColumnSchema columnSchema2 = new ColumnSchema(DataType.DECIMAL, "column", "3", true, encoders, true);
    ColumnSchema columnSchema3 = new ColumnSchema(DataType.DOUBLE, "column", "3", true, encoders, true);
    ColumnSchema columnSchema4 = new ColumnSchema(DataType.LONG, "column", "3", true, encoders, true);
    ColumnSchema columnSchema5 = new ColumnSchema(DataType.SHORT, "column", "3", true, encoders, true);
    ColumnSchema columnSchema6 = new ColumnSchema(DataType.STRUCT, "column", "3", true, encoders, true);
    ColumnSchema columnSchema7 = new ColumnSchema(DataType.STRING, "column", "3", true, encoders, true);
    final List<ColumnSchema> columnSchemas = new ArrayList<>();
    columnSchemas.add(columnSchema);
    columnSchemas.add(columnSchema1);
    columnSchemas.add(columnSchema2);
    columnSchemas.add(columnSchema3);
    columnSchemas.add(columnSchema4);
    columnSchemas.add(columnSchema5);
    columnSchemas.add(columnSchema6);
    columnSchemas.add(columnSchema7);
    org.apache.carbondata.format.BlockletIndex blockletIndex1 = new org.apache.carbondata.format.BlockletIndex();
    List<org.apache.carbondata.format.BlockletIndex> blockletIndexArrayList = new ArrayList<>();
    blockletIndexArrayList.add(blockletIndex1);
    org.apache.carbondata.format.BlockletInfo blockletInfo = new org.apache.carbondata.format.BlockletInfo();
    List<org.apache.carbondata.format.BlockletInfo> blockletInfoArrayList = new ArrayList<>();
    blockletInfoArrayList.add(blockletInfo);
    final FileFooter fileFooter = new FileFooter(1, 3, columnSchemas, segmentInfo1, blockletIndexArrayList);
    fileFooter.setBlocklet_info_list(blockletInfoArrayList);
    BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
    blockletBTreeIndex.setStart_key("1".getBytes());
    blockletBTreeIndex.setEnd_key("3".getBytes());
    blockletIndex1.setB_tree_index(blockletBTreeIndex);
    BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();
    blockletMinMaxIndex.setMax_values(Arrays.asList(ByteBuffer.allocate(1).put((byte) 2)));
    blockletMinMaxIndex.setMin_values(Arrays.asList(ByteBuffer.allocate(1).put((byte) 1)));
    blockletIndex1.setMin_max_index(blockletMinMaxIndex);
    new MockUp<FileFactory>() {

        @SuppressWarnings("unused")
        @Mock
        public FileFactory.FileType getFileType(String path) {
            return FileFactory.FileType.LOCAL;
        }

        @SuppressWarnings("unused")
        @Mock
        public FileHolder getFileHolder(FileFactory.FileType fileType) {
            return new FileHolderImpl();
        }
    };
    new MockUp<FileHolderImpl>() {

        @SuppressWarnings("unused")
        @Mock
        public long readLong(String filePath, long offset) {
            return 1;
        }
    };
    new MockUp<CarbonFooterReader>() {

        @SuppressWarnings("unused")
        @Mock
        public FileFooter readFooter() throws IOException {
            return fileFooter;
        }
    };
    SegmentInfo segmentInfo = new SegmentInfo();
    int[] arr = { 1, 2, 3 };
    segmentInfo.setColumnCardinality(arr);
    segmentInfo.setNumberOfColumns(segmentInfo1.getNum_cols());
    dataFileFooter.setNumberOfRows(3);
    dataFileFooter.setSegmentInfo(segmentInfo);
    TableBlockInfo info = new TableBlockInfo("/file.carbondata", 1, "0", new String[0], 1, ColumnarFormatVersion.V1);
    DataFileFooter result = dataFileFooterConverter.readDataFileFooter(info);
    assertEquals(result.getNumberOfRows(), 3);
}