Examples with CarbonMeasure org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure used on opensource projects

Example 61 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class AbstractQueryExecutor method getBlockExecutionInfoForBlock.

/**
 * Below method will be used to get the block execution info which is
 * required to execute any block  based on query model
 *
 * @param queryModel query model from user query
 * @param blockIndex block index
 * @return block execution info
 */
private BlockExecutionInfo getBlockExecutionInfoForBlock(QueryModel queryModel, AbstractIndex blockIndex, int numberOfBlockletToScan, String filePath, String[] deleteDeltaFiles, Segment segment) {
    BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
    SegmentProperties segmentProperties = blockIndex.getSegmentProperties();
    // set actual query dimensions and measures. It may differ in case of restructure scenarios
    RestructureUtil.actualProjectionOfSegment(blockExecutionInfo, queryModel, segmentProperties);
    // below is to get only those dimension in query which is present in the
    // table block
    List<ProjectionDimension> projectDimensions = RestructureUtil.createDimensionInfoAndGetCurrentBlockQueryDimension(blockExecutionInfo, blockExecutionInfo.getActualQueryDimensions(), segmentProperties.getDimensions(), segmentProperties.getComplexDimensions(), blockExecutionInfo.getActualQueryMeasures().length, queryModel.getTable().getTableInfo().isTransactionalTable(), queryModel);
    boolean isStandardTable = CarbonUtil.isStandardCarbonTable(queryModel.getTable());
    String blockId = CarbonUtil.getBlockId(queryModel.getAbsoluteTableIdentifier(), filePath, segment.getSegmentNo(), queryModel.getTable().getTableInfo().isTransactionalTable(), isStandardTable, queryModel.getTable().isHivePartitionTable());
    blockExecutionInfo.setBlockId(CarbonTablePath.getShortBlockId(blockId));
    blockExecutionInfo.setDeleteDeltaFilePath(deleteDeltaFiles);
    blockExecutionInfo.setStartBlockletIndex(0);
    blockExecutionInfo.setNumberOfBlockletToScan(numberOfBlockletToScan);
    blockExecutionInfo.setProjectionDimensions(projectDimensions.toArray(new ProjectionDimension[0]));
    // get measures present in the current block
    List<ProjectionMeasure> projectionMeasures = RestructureUtil.createMeasureInfoAndGetCurrentBlockQueryMeasures(blockExecutionInfo, blockExecutionInfo.getActualQueryMeasures(), segmentProperties.getMeasures(), queryModel.getTable().getTableInfo().isTransactionalTable(), queryModel);
    blockExecutionInfo.setProjectionMeasures(projectionMeasures.toArray(new ProjectionMeasure[projectionMeasures.size()]));
    blockExecutionInfo.setDataBlock(blockIndex);
    // setting whether raw record query or not
    blockExecutionInfo.setRawRecordDetailQuery(queryModel.isForcedDetailRawQuery());
    // total number dimension
    blockExecutionInfo.setTotalNumberDimensionToRead(segmentProperties.getDimensionOrdinalToChunkMapping().size());
    blockExecutionInfo.setReadOnlyDelta(queryModel.isReadOnlyDelta());
    if (queryModel.isReadPageByPage()) {
        blockExecutionInfo.setPrefetchBlocklet(false);
        LOGGER.info("Query prefetch is: false, read page by page");
    } else {
        LOGGER.info("Query prefetch is: " + queryModel.isPreFetchData());
        blockExecutionInfo.setPrefetchBlocklet(queryModel.isPreFetchData());
    }
    // In case of fg index it should not go to direct fill.
    boolean fgIndexPathPresent = false;
    for (TableBlockInfo blockInfo : queryModel.getTableBlockInfos()) {
        fgIndexPathPresent = blockInfo.getIndexWriterPath() != null;
        if (fgIndexPathPresent) {
            queryModel.setDirectVectorFill(false);
            break;
        }
    }
    blockExecutionInfo.setDirectVectorFill(queryModel.isDirectVectorFill());
    blockExecutionInfo.setTotalNumberOfMeasureToRead(segmentProperties.getMeasuresOrdinalToChunkMapping().size());
    blockExecutionInfo.setComplexDimensionInfoMap(QueryUtil.getComplexDimensionsMap(projectDimensions, segmentProperties.getDimensionOrdinalToChunkMapping(), queryProperties.complexFilterDimension));
    if (null != queryModel.getIndexFilter()) {
        FilterResolverIntf filterResolverIntf;
        if (!filePath.startsWith(queryModel.getTable().getTablePath())) {
            filterResolverIntf = queryModel.getIndexFilter().getExternalSegmentResolver();
        } else {
            // loading the filter executor tree for filter evaluation
            filterResolverIntf = queryModel.getIndexFilter().getResolver();
        }
        blockExecutionInfo.setFilterExecutorTree(FilterUtil.getFilterExecutorTree(filterResolverIntf, segmentProperties, blockExecutionInfo.getComplexDimensionInfoMap(), false));
    }
    // expression measure
    List<CarbonMeasure> expressionMeasures = new ArrayList<CarbonMeasure>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    // setting all the dimension chunk indexes to be read from file
    int numberOfElementToConsider = 0;
    // list of dimensions to be projected
    Set<Integer> allProjectionListDimensionIndexes = new LinkedHashSet<>();
    // create a list of filter dimensions present in the current block
    Set<CarbonDimension> currentBlockFilterDimensions = getCurrentBlockFilterDimensions(queryProperties.complexFilterDimension, segmentProperties);
    int[] dimensionChunkIndexes = QueryUtil.getDimensionChunkIndexes(projectDimensions, segmentProperties.getDimensionOrdinalToChunkMapping(), currentBlockFilterDimensions, allProjectionListDimensionIndexes);
    ReusableDataBuffer[] dimensionBuffer = new ReusableDataBuffer[reusableDimensionBufferSize];
    for (int i = 0; i < dimensionBuffer.length; i++) {
        dimensionBuffer[i] = new ReusableDataBuffer();
    }
    blockExecutionInfo.setDimensionReusableDataBuffer(dimensionBuffer);
    int numberOfColumnToBeReadInOneIO = Integer.parseInt(CarbonProperties.getInstance().getProperty(CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO, CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULT_VALUE));
    if (dimensionChunkIndexes.length > 0) {
        numberOfElementToConsider = dimensionChunkIndexes[dimensionChunkIndexes.length - 1] == segmentProperties.getBlockToDimensionOrdinalMapping().size() - 1 ? dimensionChunkIndexes.length - 1 : dimensionChunkIndexes.length;
        blockExecutionInfo.setAllSelectedDimensionColumnIndexRange(CarbonUtil.getRangeIndex(dimensionChunkIndexes, numberOfElementToConsider, numberOfColumnToBeReadInOneIO));
    } else {
        blockExecutionInfo.setAllSelectedDimensionColumnIndexRange(new int[0][0]);
    }
    // get the list of updated filter measures present in the current block
    Set<CarbonMeasure> filterMeasures = getCurrentBlockFilterMeasures(queryProperties.filterMeasures, segmentProperties);
    // list of measures to be projected
    List<Integer> allProjectionListMeasureIndexes = new ArrayList<>();
    int[] measureChunkIndexes = QueryUtil.getMeasureChunkIndexes(projectionMeasures, expressionMeasures, segmentProperties.getMeasuresOrdinalToChunkMapping(), filterMeasures, allProjectionListMeasureIndexes);
    ReusableDataBuffer[] measureBuffer = new ReusableDataBuffer[allProjectionListMeasureIndexes.size()];
    for (int i = 0; i < measureBuffer.length; i++) {
        measureBuffer[i] = new ReusableDataBuffer();
    }
    blockExecutionInfo.setMeasureReusableDataBuffer(measureBuffer);
    if (measureChunkIndexes.length > 0) {
        numberOfElementToConsider = measureChunkIndexes[measureChunkIndexes.length - 1] == segmentProperties.getMeasures().size() - 1 ? measureChunkIndexes.length - 1 : measureChunkIndexes.length;
        // setting all the measure chunk indexes to be read from file
        blockExecutionInfo.setAllSelectedMeasureIndexRange(CarbonUtil.getRangeIndex(measureChunkIndexes, numberOfElementToConsider, numberOfColumnToBeReadInOneIO));
    } else {
        blockExecutionInfo.setAllSelectedMeasureIndexRange(new int[0][0]);
    }
    // setting the indexes of list of dimension in projection list
    blockExecutionInfo.setProjectionListDimensionIndexes(ArrayUtils.toPrimitive(allProjectionListDimensionIndexes.toArray(new Integer[allProjectionListDimensionIndexes.size()])));
    // setting the indexes of list of measures in projection list
    blockExecutionInfo.setProjectionListMeasureIndexes(ArrayUtils.toPrimitive(allProjectionListMeasureIndexes.toArray(new Integer[allProjectionListMeasureIndexes.size()])));
    // setting the size of fixed key column (dictionary column)
    blockExecutionInfo.setFixedLengthKeySize(getKeySize(projectDimensions, segmentProperties));
    List<Integer> dictionaryColumnChunkIndex = new ArrayList<Integer>();
    List<Integer> noDictionaryColumnChunkIndex = new ArrayList<Integer>();
    // get the block index to be read from file for query dimension
    // for both dictionary columns and no dictionary columns
    QueryUtil.fillQueryDimensionChunkIndexes(projectDimensions, segmentProperties.getDimensionOrdinalToChunkMapping(), dictionaryColumnChunkIndex, noDictionaryColumnChunkIndex);
    int[] queryDictionaryColumnChunkIndexes = ArrayUtils.toPrimitive(dictionaryColumnChunkIndex.toArray(new Integer[dictionaryColumnChunkIndex.size()]));
    // column key will be filled based on key order
    if (!queryModel.isForcedDetailRawQuery()) {
        Arrays.sort(queryDictionaryColumnChunkIndexes);
    }
    blockExecutionInfo.setDictionaryColumnChunkIndex(queryDictionaryColumnChunkIndexes);
    // setting the no dictionary column block indexes
    blockExecutionInfo.setNoDictionaryColumnChunkIndexes(ArrayUtils.toPrimitive(noDictionaryColumnChunkIndex.toArray(new Integer[noDictionaryColumnChunkIndex.size()])));
    blockExecutionInfo.setComplexColumnParentBlockIndexes(getComplexDimensionParentBlockIndexes(projectDimensions));
    blockExecutionInfo.setVectorBatchCollector(queryModel.isVectorReader());
    DataTypeUtil.setDataTypeConverter(queryModel.getConverter());
    blockExecutionInfo.setRequiredRowId(queryModel.isRequiredRowId());
    return blockExecutionInfo;
}

Example 62 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class RestructureBasedVectorResultCollector method fillDataForNonExistingMeasures.

/**
 * This method will fill the default values of non existing measures in the current block
 */
private void fillDataForNonExistingMeasures() {
    for (int i = 0; i < executionInfo.getActualQueryMeasures().length; i++) {
        if (!measureInfo.getMeasureExists()[i]) {
            int queryOrder = executionInfo.getActualQueryMeasures()[i].getOrdinal();
            CarbonMeasure measure = executionInfo.getActualQueryMeasures()[i].getMeasure();
            ColumnVectorInfo columnVectorInfo = allColumnInfo[queryOrder];
            CarbonColumnVector vector = columnVectorInfo.vector;
            Object defaultValue = measureDefaultValues[i];
            if (null == defaultValue) {
                vector.putNulls(columnVectorInfo.vectorOffset, columnVectorInfo.size);
            } else {
                DataType dataType = measureInfo.getMeasureDataTypes()[i];
                if (dataType == DataTypes.SHORT) {
                    vector.putShorts(columnVectorInfo.vectorOffset, columnVectorInfo.size, (short) defaultValue);
                } else if (dataType == DataTypes.INT) {
                    vector.putInts(columnVectorInfo.vectorOffset, columnVectorInfo.size, (int) defaultValue);
                } else if (dataType == DataTypes.LONG) {
                    vector.putLongs(columnVectorInfo.vectorOffset, columnVectorInfo.size, (long) defaultValue);
                } else if (DataTypes.isDecimal(dataType)) {
                    vector.putDecimals(columnVectorInfo.vectorOffset, columnVectorInfo.size, (BigDecimal) defaultValue, measure.getPrecision());
                } else if (dataType == DataTypes.BOOLEAN) {
                    vector.putBoolean(columnVectorInfo.vectorOffset, (Boolean) defaultValue);
                } else {
                    vector.putDoubles(columnVectorInfo.vectorOffset, columnVectorInfo.size, (double) defaultValue);
                }
            }
        }
    }
}

Example 63 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class SegmentProperties method fillDimensionAndMeasureDetails.

/**
 * below method will fill dimension and measure detail of the block.
 *
 * @param columnsInTable
 */
private void fillDimensionAndMeasureDetails(List<ColumnSchema> columnsInTable) {
    ColumnSchema columnSchema = null;
    // ordinal will be required to read the data from file block
    int dimensionOrdinal = 0;
    int measureOrdinal = -1;
    // table ordinal is actually a schema ordinal this is required as
    // cardinality array
    // which is stored in segment info contains -1 if that particular column
    // is n
    boolean isComplexDimensionStarted = false;
    CarbonDimension carbonDimension = null;
    int keyOrdinal = 0;
    int counter = 0;
    int complexTypeOrdinal = -1;
    while (counter < columnsInTable.size()) {
        columnSchema = columnsInTable.get(counter);
        if (columnSchema.isDimensionColumn()) {
            // column as it was not the part of MDKey
            if (CarbonUtil.hasEncoding(columnSchema.getEncodingList(), Encoding.DICTIONARY) && !isComplexDimensionStarted && columnSchema.getNumberOfChild() == 0) {
                this.numberOfDictDimensions++;
                this.numberOfColumnsAfterFlatten++;
                if (columnSchema.isSortColumn()) {
                    this.numberOfSortColumns++;
                }
                // if it is a columnar dimension participated in MDKey then added
                // key ordinal and dimension ordinal
                carbonDimension = new CarbonDimension(columnSchema, dimensionOrdinal++, keyOrdinal++, -1);
            } else // will be added to complex type
            if (isComplexDimensionStarted || columnSchema.getDataType().isComplexType()) {
                carbonDimension = new CarbonDimension(columnSchema, dimensionOrdinal++, -1, ++complexTypeOrdinal);
                carbonDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
                complexDimensions.add(carbonDimension);
                isComplexDimensionStarted = true;
                dimensionOrdinal = readAllComplexTypeChildren(dimensionOrdinal, columnSchema.getNumberOfChild(), columnsInTable, carbonDimension, complexTypeOrdinal);
                counter = dimensionOrdinal;
                complexTypeOrdinal = assignComplexOrdinal(carbonDimension, complexTypeOrdinal);
                this.numberOfColumnsAfterFlatten += getNumColumnsAfterFlatten(carbonDimension);
                continue;
            } else {
                // for no dictionary dimension
                carbonDimension = new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1);
                numberOfColumnsAfterFlatten++;
                numberOfNoDictionaryDimension++;
                if (columnSchema.isSortColumn()) {
                    this.numberOfSortColumns++;
                }
            }
            dimensions.add(carbonDimension);
        } else {
            numberOfColumnsAfterFlatten++;
            measures.add(new CarbonMeasure(columnSchema, ++measureOrdinal));
        }
        counter++;
    }
    lastDimensionColOrdinal = dimensionOrdinal;
}

Example 64 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class SegmentProperties method createColumnValueLength.

/**
 * Return column value length in byte for all columns in the table
 * for dimension and complex column it is -1 (for DATE it is 4),
 * for measure is 8 (for decimal is -1)
 */
public int[] createColumnValueLength() {
    int[] length = new int[numberOfColumnsAfterFlatten];
    int index = 0;
    for (CarbonDimension dimension : dimensions) {
        DataType dataType = dimension.getDataType();
        if (dataType == DataTypes.DATE) {
            length[index] = 4;
        } else {
            length[index] = -1;
        }
        index++;
    }
    for (CarbonDimension complexDimension : complexDimensions) {
        int depth = getNumColumnsAfterFlatten(complexDimension);
        for (int i = 0; i < depth; i++) {
            length[index++] = -1;
        }
    }
    for (CarbonMeasure measure : measures) {
        DataType dataType = measure.getDataType();
        if (DataTypes.isDecimal(dataType)) {
            length[index++] = -1;
        } else {
            length[index++] = 8;
        }
    }
    return length;
}

Example 65 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class TableSpec method addMeasures.

private void addMeasures(List<CarbonMeasure> measures) {
    for (int i = 0; i < measures.size(); i++) {
        CarbonMeasure measure = measures.get(i);
        measureSpec[i] = new MeasureSpec(measure.getColName(), measure.getDataType());
    }
}