Examples with BlockletDetailInfo org.apache.carbondata.core.indexstore.BlockletDetailInfo used on opensource projects

Example 1 with BlockletDetailInfo

use of org.apache.carbondata.core.indexstore.BlockletDetailInfo in project carbondata by apache.

the class AbstractQueryExecutor method initQuery.

/**
 * Below method will be used to fill the executor properties based on query
 * model it will parse the query model and get the detail and fill it in
 * query properties
 *
 * @param queryModel
 */
protected void initQuery(QueryModel queryModel) throws IOException {
    StandardLogService.setThreadName(StandardLogService.getPartitionID(queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName()), queryModel.getQueryId());
    LOGGER.info("Query will be executed on table: " + queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName());
    // add executor service for query execution
    queryProperties.executorService = Executors.newCachedThreadPool();
    // Initializing statistics list to record the query statistics
    // creating copy on write to handle concurrent scenario
    queryProperties.queryStatisticsRecorder = CarbonTimeStatisticsFactory.createExecutorRecorder(queryModel.getQueryId());
    queryModel.setStatisticsRecorder(queryProperties.queryStatisticsRecorder);
    QueryStatistic queryStatistic = new QueryStatistic();
    // sort the block info
    // so block will be loaded in sorted order this will be required for
    // query execution
    Collections.sort(queryModel.getTableBlockInfos());
    if (queryModel.getTableBlockInfos().get(0).getDetailInfo() != null) {
        List<AbstractIndex> indexList = new ArrayList<>();
        Map<String, List<TableBlockInfo>> listMap = new LinkedHashMap<>();
        for (TableBlockInfo blockInfo : queryModel.getTableBlockInfos()) {
            List<TableBlockInfo> tableBlockInfos = listMap.get(blockInfo.getFilePath());
            if (tableBlockInfos == null) {
                tableBlockInfos = new ArrayList<>();
                listMap.put(blockInfo.getFilePath(), tableBlockInfos);
            }
            BlockletDetailInfo blockletDetailInfo = blockInfo.getDetailInfo();
            // the blocklet information from block file
            if (blockletDetailInfo.getBlockletInfo() == null) {
                readAndFillBlockletInfo(blockInfo, tableBlockInfos, blockletDetailInfo);
            } else {
                tableBlockInfos.add(blockInfo);
            }
        }
        for (List<TableBlockInfo> tableBlockInfos : listMap.values()) {
            indexList.add(new IndexWrapper(tableBlockInfos));
        }
        queryProperties.dataBlocks = indexList;
    } else {
        // get the table blocks
        CacheProvider cacheProvider = CacheProvider.getInstance();
        BlockIndexStore<TableBlockUniqueIdentifier, AbstractIndex> cache = (BlockIndexStore) cacheProvider.createCache(CacheType.EXECUTOR_BTREE);
        // remove the invalid table blocks, block which is deleted or compacted
        cache.removeTableBlocks(queryModel.getInvalidSegmentIds(), queryModel.getAbsoluteTableIdentifier());
        List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers = prepareTableBlockUniqueIdentifier(queryModel.getTableBlockInfos(), queryModel.getAbsoluteTableIdentifier());
        cache.removeTableBlocksIfHorizontalCompactionDone(queryModel);
        queryProperties.dataBlocks = cache.getAll(tableBlockUniqueIdentifiers);
    }
    queryStatistic.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, System.currentTimeMillis());
    queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
    // calculating the total number of aggeragted columns
    int measureCount = queryModel.getProjectionMeasures().size();
    int currentIndex = 0;
    DataType[] dataTypes = new DataType[measureCount];
    for (ProjectionMeasure carbonMeasure : queryModel.getProjectionMeasures()) {
        // adding the data type and aggregation type of all the measure this
        // can be used
        // to select the aggregator
        dataTypes[currentIndex] = carbonMeasure.getMeasure().getDataType();
        currentIndex++;
    }
    queryProperties.measureDataTypes = dataTypes;
    // as aggregation will be executed in following order
    // 1.aggregate dimension expression
    // 2. expression
    // 3. query measure
    // so calculating the index of the expression start index
    // and measure column start index
    queryProperties.filterMeasures = new HashSet<>();
    queryProperties.complexFilterDimension = new HashSet<>();
    QueryUtil.getAllFilterDimensions(queryModel.getFilterExpressionResolverTree(), queryProperties.complexFilterDimension, queryProperties.filterMeasures);
    CarbonTable carbonTable = queryModel.getTable();
    TableProvider tableProvider = new SingleTableProvider(carbonTable);
    queryStatistic = new QueryStatistic();
    // dictionary column unique column id to dictionary mapping
    // which will be used to get column actual data
    queryProperties.columnToDictionaryMapping = QueryUtil.getDimensionDictionaryDetail(queryModel.getProjectionDimensions(), queryProperties.complexFilterDimension, queryModel.getAbsoluteTableIdentifier(), tableProvider);
    queryStatistic.addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY, System.currentTimeMillis());
    queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
    queryModel.setColumnToDictionaryMapping(queryProperties.columnToDictionaryMapping);
}

Example 2 with BlockletDetailInfo

use of org.apache.carbondata.core.indexstore.BlockletDetailInfo in project carbondata by apache.

the class AbstractQueryExecutor method readAndFillBlockletInfo.

/**
 * Read the file footer of block file and get the blocklets to query
 */
private void readAndFillBlockletInfo(TableBlockInfo blockInfo, List<TableBlockInfo> tableBlockInfos, BlockletDetailInfo blockletDetailInfo) throws IOException {
    blockInfo.setBlockOffset(blockletDetailInfo.getBlockFooterOffset());
    blockInfo.setDetailInfo(null);
    DataFileFooter fileFooter = CarbonUtil.readMetadatFile(blockInfo);
    blockInfo.setDetailInfo(blockletDetailInfo);
    List<BlockletInfo> blockletList = fileFooter.getBlockletList();
    short count = 0;
    for (BlockletInfo blockletInfo : blockletList) {
        TableBlockInfo info = blockInfo.copy();
        BlockletDetailInfo detailInfo = info.getDetailInfo();
        detailInfo.setRowCount(blockletInfo.getNumberOfRows());
        // update min and max values in case of old store for measures as min and max is written
        // opposite for measures in old store
        byte[][] maxValues = CarbonUtil.updateMinMaxValues(fileFooter, blockletInfo.getBlockletIndex().getMinMaxIndex().getMaxValues(), blockletInfo.getBlockletIndex().getMinMaxIndex().getMinValues(), false);
        byte[][] minValues = CarbonUtil.updateMinMaxValues(fileFooter, blockletInfo.getBlockletIndex().getMinMaxIndex().getMaxValues(), blockletInfo.getBlockletIndex().getMinMaxIndex().getMinValues(), true);
        blockletInfo.getBlockletIndex().getMinMaxIndex().setMaxValues(maxValues);
        blockletInfo.getBlockletIndex().getMinMaxIndex().setMinValues(minValues);
        detailInfo.setBlockletInfo(blockletInfo);
        detailInfo.setPagesCount((short) blockletInfo.getNumberOfPages());
        detailInfo.setBlockletId(count);
        info.setDataBlockFromOldStore(true);
        tableBlockInfos.add(info);
        count++;
    }
}

Example 3 with BlockletDetailInfo

use of org.apache.carbondata.core.indexstore.BlockletDetailInfo in project carbondata by apache.

the class CarbonInputSplit method readFields.

@Override
public void readFields(DataInput in) throws IOException {
    super.readFields(in);
    this.segmentId = in.readUTF();
    this.version = ColumnarFormatVersion.valueOf(in.readShort());
    this.bucketId = in.readUTF();
    this.blockletId = in.readUTF();
    int numInvalidSegment = in.readInt();
    invalidSegments = new ArrayList<>(numInvalidSegment);
    for (int i = 0; i < numInvalidSegment; i++) {
        invalidSegments.add(in.readUTF());
    }
    int numberOfDeleteDeltaFiles = in.readInt();
    deleteDeltaFiles = new String[numberOfDeleteDeltaFiles];
    for (int i = 0; i < numberOfDeleteDeltaFiles; i++) {
        deleteDeltaFiles[i] = in.readUTF();
    }
    boolean detailInfoExists = in.readBoolean();
    if (detailInfoExists) {
        detailInfo = new BlockletDetailInfo();
        detailInfo.readFields(in);
    }
    boolean dataMapWriterPathExists = in.readBoolean();
    if (dataMapWriterPathExists) {
        dataMapWritePath = in.readUTF();
    }
}

Example 4 with BlockletDetailInfo

use of org.apache.carbondata.core.indexstore.BlockletDetailInfo in project carbondata by apache.

the class CarbonLocalInputSplit method convertSplit.

public static CarbonInputSplit convertSplit(CarbonLocalInputSplit carbonLocalInputSplit) {
    CarbonInputSplit inputSplit = new CarbonInputSplit(carbonLocalInputSplit.getSegmentId(), "0", new Path(carbonLocalInputSplit.getPath()), carbonLocalInputSplit.getStart(), carbonLocalInputSplit.getLength(), carbonLocalInputSplit.getLocations().toArray(new String[carbonLocalInputSplit.getLocations().size()]), carbonLocalInputSplit.getNumberOfBlocklets(), ColumnarFormatVersion.valueOf(carbonLocalInputSplit.getVersion()), carbonLocalInputSplit.getDeleteDeltaFiles());
    Gson gson = new Gson();
    BlockletDetailInfo blockletDetailInfo = gson.fromJson(carbonLocalInputSplit.detailInfo, BlockletDetailInfo.class);
    try {
        blockletDetailInfo.readColumnSchema(blockletDetailInfo.getColumnSchemaBinary());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    inputSplit.setDetailInfo(blockletDetailInfo);
    return inputSplit;
}

Example 5 with BlockletDetailInfo

use of org.apache.carbondata.core.indexstore.BlockletDetailInfo in project carbondata by apache.

the class CarbonUtil method getDataFileFooter.

private static DataFileFooter getDataFileFooter(TableBlockInfo tableBlockInfo, boolean forceReadDataFileFooter) throws IOException {
    BlockletDetailInfo detailInfo = tableBlockInfo.getDetailInfo();
    if (detailInfo == null || forceReadDataFileFooter) {
        AbstractDataFileFooterConverter fileFooterConverter = DataFileFooterConverterFactory.getInstance().getDataFileFooterConverter(tableBlockInfo.getVersion());
        return fileFooterConverter.readDataFileFooter(tableBlockInfo);
    } else {
        DataFileFooter fileFooter = new DataFileFooter();
        fileFooter.setSchemaUpdatedTimeStamp(detailInfo.getSchemaUpdatedTimeStamp());
        ColumnarFormatVersion version = ColumnarFormatVersion.valueOf(detailInfo.getVersionNumber());
        AbstractDataFileFooterConverter dataFileFooterConverter = DataFileFooterConverterFactory.getInstance().getDataFileFooterConverter(version);
        List<ColumnSchema> schema = dataFileFooterConverter.getSchema(tableBlockInfo);
        fileFooter.setColumnInTable(schema);
        SegmentInfo segmentInfo = new SegmentInfo();
        segmentInfo.setColumnCardinality(detailInfo.getDimLens());
        fileFooter.setSegmentInfo(segmentInfo);
        return fileFooter;
    }
}