Examples with DataFileFooter org.apache.carbondata.core.metadata.blocklet.DataFileFooter used on opensource projects

Example 26 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class CarbonCompactionExecutor method processTableBlocks.

/**
   * For processing of the table blocks.
   *
   * @return List of Carbon iterators
   */
public List<RawResultIterator> processTableBlocks() throws QueryExecutionException, IOException {
    List<RawResultIterator> resultList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    List<TableBlockInfo> list = null;
    queryModel = prepareQueryModel(list);
    // iterate each seg ID
    for (Map.Entry<String, TaskBlockInfo> taskMap : segmentMapping.entrySet()) {
        String segmentId = taskMap.getKey();
        List<DataFileFooter> listMetadata = dataFileMetadataSegMapping.get(segmentId);
        SegmentProperties sourceSegProperties = getSourceSegmentProperties(listMetadata);
        // for each segment get taskblock info
        TaskBlockInfo taskBlockInfo = taskMap.getValue();
        Set<String> taskBlockListMapping = taskBlockInfo.getTaskSet();
        for (String task : taskBlockListMapping) {
            list = taskBlockInfo.getTableBlockInfoList(task);
            Collections.sort(list);
            LOGGER.info("for task -" + task + "-block size is -" + list.size());
            queryModel.setTableBlockInfos(list);
            resultList.add(new RawResultIterator(executeBlockList(list), sourceSegProperties, destinationSegProperties));
        }
    }
    return resultList;
}

Example 27 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class SegmentTaskIndex method buildIndex.

/**
   * Below method is store the blocks in some data structure
   *
   */
public void buildIndex(List<DataFileFooter> footerList) {
    // create a segment builder info
    // in case of segment create we do not need any file path and each column value size
    // as Btree will be build as per min max and start key
    BTreeBuilderInfo btreeBuilderInfo = new BTreeBuilderInfo(footerList, null);
    BtreeBuilder blocksBuilder = new BlockBTreeBuilder();
    // load the metadata
    blocksBuilder.build(btreeBuilderInfo);
    dataRefNode = blocksBuilder.get();
    for (DataFileFooter footer : footerList) {
        totalNumberOfRows += footer.getNumberOfRows();
    }
}

Example 28 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class AbstractBlockIndexStoreCache method checkAndLoadTableBlocks.

/**
   * This method will get the value for the given key. If value does not exist
   * for the given key, it will check and load the value.
   *
   * @param tableBlock
   * @param tableBlockUniqueIdentifier
   * @param lruCacheKey
   */
protected void checkAndLoadTableBlocks(AbstractIndex tableBlock, TableBlockUniqueIdentifier tableBlockUniqueIdentifier, String lruCacheKey) throws IOException {
    // calculate the required size is
    TableBlockInfo blockInfo = tableBlockUniqueIdentifier.getTableBlockInfo();
    long requiredMetaSize = CarbonUtil.calculateMetaSize(blockInfo);
    if (requiredMetaSize > 0) {
        tableBlock.setMemorySize(requiredMetaSize);
        // load table blocks data
        // getting the data file meta data of the block
        DataFileFooter footer = CarbonUtil.readMetadatFile(blockInfo);
        footer.setBlockInfo(new BlockInfo(blockInfo));
        // building the block
        tableBlock.buildIndex(Collections.singletonList(footer));
        tableBlock.incrementAccessCount();
        boolean isTableBlockAddedToLruCache = lruCache.put(lruCacheKey, tableBlock, requiredMetaSize);
        if (!isTableBlockAddedToLruCache) {
            throw new IndexBuilderException("Cannot load table blocks into memory. Not enough memory available");
        }
    } else {
        throw new IndexBuilderException("Invalid carbon data file: " + blockInfo.getFilePath());
    }
}

Example 29 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class SegmentTaskIndexStoreTest method getDataFileFooters.

private List<DataFileFooter> getDataFileFooters() {
    SegmentInfo segmentInfo = new SegmentInfo();
    DataFileFooter footer = new DataFileFooter();
    ColumnSchema columnSchema = new ColumnSchema();
    BlockletInfo blockletInfo = new BlockletInfo();
    List<DataFileFooter> footerList = new ArrayList<DataFileFooter>();
    List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
    columnSchema.setColumnName("employeeName");
    columnSchemaList.add(new ColumnSchema());
    footer.setSegmentInfo(segmentInfo);
    footer.setColumnInTable(columnSchemaList);
    footer.setBlockletList(Arrays.asList(blockletInfo));
    footerList.add(footer);
    return footerList;
}