Examples with ProjectionDimension org.apache.carbondata.core.scan.model.ProjectionDimension used on opensource projects

Example 11 with ProjectionDimension

use of org.apache.carbondata.core.scan.model.ProjectionDimension in project carbondata by apache.

the class RestructureUtilTest method testToGetUpdatedQueryDimension.

@Test
public void testToGetUpdatedQueryDimension() {
    BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
    List<Encoding> encodingList = new ArrayList<Encoding>();
    encodingList.add(Encoding.DICTIONARY);
    ColumnSchema columnSchema1 = new ColumnSchema();
    columnSchema1.setColumnName("Id");
    columnSchema1.setDataType(DataTypes.STRING);
    columnSchema1.setColumnUniqueId(UUID.randomUUID().toString());
    columnSchema1.setEncodingList(encodingList);
    ColumnSchema columnSchema2 = new ColumnSchema();
    columnSchema2.setColumnName("Name");
    columnSchema2.setDataType(DataTypes.STRING);
    columnSchema2.setColumnUniqueId(UUID.randomUUID().toString());
    columnSchema2.setEncodingList(encodingList);
    ColumnSchema columnSchema3 = new ColumnSchema();
    columnSchema3.setColumnName("Age");
    columnSchema3.setDataType(DataTypes.INT);
    columnSchema3.setColumnUniqueId(UUID.randomUUID().toString());
    columnSchema3.setEncodingList(encodingList);
    ColumnSchema columnSchema4 = new ColumnSchema();
    columnSchema4.setColumnName("Salary");
    columnSchema4.setDataType(DataTypes.INT);
    columnSchema4.setColumnUniqueId(UUID.randomUUID().toString());
    columnSchema4.setEncodingList(encodingList);
    ColumnSchema columnSchema5 = new ColumnSchema();
    columnSchema5.setColumnName("Address");
    columnSchema5.setDataType(DataTypes.STRING);
    columnSchema5.setColumnUniqueId(UUID.randomUUID().toString());
    columnSchema5.setEncodingList(encodingList);
    CarbonDimension tableBlockDimension1 = new CarbonDimension(columnSchema1, 1, 1, 1, 1);
    CarbonDimension tableBlockDimension2 = new CarbonDimension(columnSchema2, 5, 5, 5, 5);
    List<CarbonDimension> tableBlockDimensions = Arrays.asList(tableBlockDimension1, tableBlockDimension2);
    CarbonDimension tableComplexDimension1 = new CarbonDimension(columnSchema3, 4, 4, 4, 4);
    CarbonDimension tableComplexDimension2 = new CarbonDimension(columnSchema4, 2, 2, 2, 2);
    List<CarbonDimension> tableComplexDimensions = Arrays.asList(tableComplexDimension1, tableComplexDimension2);
    ProjectionDimension queryDimension1 = new ProjectionDimension(tableBlockDimension1);
    ProjectionDimension queryDimension2 = new ProjectionDimension(tableComplexDimension2);
    ProjectionDimension queryDimension3 = new ProjectionDimension(new CarbonDimension(columnSchema5, 3, 3, 3, 3));
    ProjectionMeasure queryMeasure1 = new ProjectionMeasure(new CarbonMeasure(columnSchema3, 2));
    ProjectionMeasure queryMeasure2 = new ProjectionMeasure(new CarbonMeasure(columnSchema4, 4));
    List<ProjectionMeasure> queryMeasures = Arrays.asList(queryMeasure1, queryMeasure2);
    List<ProjectionDimension> queryDimensions = Arrays.asList(queryDimension1, queryDimension2, queryDimension3);
    List<ProjectionDimension> result = null;
    result = RestructureUtil.createDimensionInfoAndGetCurrentBlockQueryDimension(blockExecutionInfo, queryDimensions, tableBlockDimensions, tableComplexDimensions, queryMeasures.size());
    List<CarbonDimension> resultDimension = new ArrayList<>(result.size());
    for (ProjectionDimension queryDimension : result) {
        resultDimension.add(queryDimension.getDimension());
    }
    assertThat(resultDimension, is(equalTo(Arrays.asList(queryDimension1.getDimension(), queryDimension2.getDimension()))));
}

Example 12 with ProjectionDimension

use of org.apache.carbondata.core.scan.model.ProjectionDimension in project carbondata by apache.

the class RestructureBasedRawResultCollector method initRestructuredKeyGenerator.

/**
 * This method will create a new key generator for generating mdKey according to latest schema
 */
private void initRestructuredKeyGenerator() {
    SegmentProperties segmentProperties = executionInfo.getDataBlock().getSegmentProperties();
    ProjectionDimension[] queryDimensions = executionInfo.getActualQueryDimensions();
    List<Integer> updatedColumnCardinality = new ArrayList<>(queryDimensions.length);
    List<Integer> updatedDimensionPartitioner = new ArrayList<>(queryDimensions.length);
    int[] dictionaryColumnBlockIndex = executionInfo.getDictionaryColumnChunkIndex();
    int dimCounterInCurrentBlock = 0;
    for (int i = 0; i < queryDimensions.length; i++) {
        if (queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) {
            if (executionInfo.getDimensionInfo().getDimensionExists()[i]) {
                // get the dictionary key ordinal as column cardinality in segment properties
                // will only be for dictionary encoded columns
                CarbonDimension currentBlockDimension = segmentProperties.getDimensions().get(dictionaryColumnBlockIndex[dimCounterInCurrentBlock]);
                updatedColumnCardinality.add(segmentProperties.getDimColumnsCardinality()[currentBlockDimension.getKeyOrdinal()]);
                updatedDimensionPartitioner.add(segmentProperties.getDimensionPartitions()[currentBlockDimension.getKeyOrdinal()]);
                dimCounterInCurrentBlock++;
            } else {
                // partitioner index will be 1 every column will be in columnar format
                updatedDimensionPartitioner.add(1);
                // for direct dictionary 4 bytes need to be allocated else 1
                if (queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
                    updatedColumnCardinality.add(Integer.MAX_VALUE);
                } else {
                    // cardinality will be 2 will user has provided a default value
                    byte[] defaultValue = queryDimensions[i].getDimension().getDefaultValue();
                    if (null != defaultValue) {
                        updatedColumnCardinality.add(CarbonCommonConstants.DICTIONARY_DEFAULT_CARDINALITY + 1);
                    } else {
                        updatedColumnCardinality.add(CarbonCommonConstants.DICTIONARY_DEFAULT_CARDINALITY);
                    }
                }
            }
        }
    }
    if (!updatedColumnCardinality.isEmpty()) {
        int[] latestColumnCardinality = ArrayUtils.toPrimitive(updatedColumnCardinality.toArray(new Integer[updatedColumnCardinality.size()]));
        int[] latestColumnPartitioner = ArrayUtils.toPrimitive(updatedDimensionPartitioner.toArray(new Integer[updatedDimensionPartitioner.size()]));
        int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(latestColumnCardinality, latestColumnPartitioner);
        restructuredKeyGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
    }
}

Example 13 with ProjectionDimension

use of org.apache.carbondata.core.scan.model.ProjectionDimension in project carbondata by apache.

the class RestructureUtil method createDimensionInfoAndGetCurrentBlockQueryDimension.

/**
 * Below method will be used to get the updated query dimension updation
 * means, after restructuring some dimension will be not present in older
 * table blocks in that case we need to select only those dimension out of
 * query dimension which is present in the current table block
 *
 * @param blockExecutionInfo
 * @param queryDimensions
 * @param tableBlockDimensions
 * @param tableComplexDimension
 * @return list of query dimension which is present in the table block
 */
public static List<ProjectionDimension> createDimensionInfoAndGetCurrentBlockQueryDimension(BlockExecutionInfo blockExecutionInfo, List<ProjectionDimension> queryDimensions, List<CarbonDimension> tableBlockDimensions, List<CarbonDimension> tableComplexDimension, int measureCount) {
    List<ProjectionDimension> presentDimension = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    boolean[] isDimensionExists = new boolean[queryDimensions.size()];
    Object[] defaultValues = new Object[queryDimensions.size()];
    // create dimension information instance
    DimensionInfo dimensionInfo = new DimensionInfo(isDimensionExists, defaultValues);
    dimensionInfo.dataType = new DataType[queryDimensions.size() + measureCount];
    int newDictionaryColumnCount = 0;
    int newNoDictionaryColumnCount = 0;
    // selecting only those dimension which is present in the query
    int dimIndex = 0;
    for (ProjectionDimension queryDimension : queryDimensions) {
        if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) {
            presentDimension.add(queryDimension);
            isDimensionExists[dimIndex] = true;
            dimensionInfo.dataType[queryDimension.getOrdinal()] = queryDimension.getDimension().getDataType();
        } else {
            for (CarbonDimension tableDimension : tableBlockDimensions) {
                if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) {
                    ProjectionDimension currentBlockDimension = new ProjectionDimension(tableDimension);
                    tableDimension.getColumnSchema().setPrecision(queryDimension.getDimension().getColumnSchema().getPrecision());
                    tableDimension.getColumnSchema().setScale(queryDimension.getDimension().getColumnSchema().getScale());
                    tableDimension.getColumnSchema().setDefaultValue(queryDimension.getDimension().getDefaultValue());
                    currentBlockDimension.setOrdinal(queryDimension.getOrdinal());
                    presentDimension.add(currentBlockDimension);
                    isDimensionExists[dimIndex] = true;
                    dimensionInfo.dataType[currentBlockDimension.getOrdinal()] = currentBlockDimension.getDimension().getDataType();
                    break;
                }
            }
            // if dimension is found then no need to search in the complex dimensions list
            if (isDimensionExists[dimIndex]) {
                dimIndex++;
                continue;
            }
            for (CarbonDimension tableDimension : tableComplexDimension) {
                if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) {
                    ProjectionDimension currentBlockDimension = new ProjectionDimension(tableDimension);
                    // TODO: for complex dimension set scale and precision by traversing
                    // the child dimensions
                    currentBlockDimension.setOrdinal(queryDimension.getOrdinal());
                    presentDimension.add(currentBlockDimension);
                    isDimensionExists[dimIndex] = true;
                    dimensionInfo.dataType[currentBlockDimension.getOrdinal()] = currentBlockDimension.getDimension().getDataType();
                    break;
                }
            }
            // add default value only in case query dimension is not found in the current block
            if (!isDimensionExists[dimIndex]) {
                defaultValues[dimIndex] = validateAndGetDefaultValue(queryDimension.getDimension());
                blockExecutionInfo.setRestructuredBlock(true);
                // newly added columns data need to be filled
                if (queryDimension.getDimension().hasEncoding(Encoding.DICTIONARY)) {
                    dimensionInfo.setDictionaryColumnAdded(true);
                    newDictionaryColumnCount++;
                } else {
                    dimensionInfo.setNoDictionaryColumnAdded(true);
                    newNoDictionaryColumnCount++;
                }
            }
        }
        dimIndex++;
    }
    dimensionInfo.setNewDictionaryColumnCount(newDictionaryColumnCount);
    dimensionInfo.setNewNoDictionaryColumnCount(newNoDictionaryColumnCount);
    blockExecutionInfo.setDimensionInfo(dimensionInfo);
    return presentDimension;
}

Example 14 with ProjectionDimension

use of org.apache.carbondata.core.scan.model.ProjectionDimension in project carbondata by apache.

the class AbstractQueryExecutor method getBlockExecutionInfoForBlock.

/**
 * Below method will be used to get the block execution info which is
 * required to execute any block  based on query model
 *
 * @param queryModel query model from user query
 * @param blockIndex block index
 * @return block execution info
 * @throws QueryExecutionException any failure during block info creation
 */
private BlockExecutionInfo getBlockExecutionInfoForBlock(QueryModel queryModel, AbstractIndex blockIndex, int startBlockletIndex, int numberOfBlockletToScan, String filePath, String[] deleteDeltaFiles, String segmentId) throws QueryExecutionException {
    BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
    SegmentProperties segmentProperties = blockIndex.getSegmentProperties();
    List<CarbonDimension> tableBlockDimensions = segmentProperties.getDimensions();
    // below is to get only those dimension in query which is present in the
    // table block
    List<ProjectionDimension> projectDimensions = RestructureUtil.createDimensionInfoAndGetCurrentBlockQueryDimension(blockExecutionInfo, queryModel.getProjectionDimensions(), tableBlockDimensions, segmentProperties.getComplexDimensions(), queryModel.getProjectionMeasures().size());
    blockExecutionInfo.setBlockId(CarbonUtil.getBlockId(queryModel.getAbsoluteTableIdentifier(), filePath, segmentId));
    blockExecutionInfo.setDeleteDeltaFilePath(deleteDeltaFiles);
    blockExecutionInfo.setStartBlockletIndex(startBlockletIndex);
    blockExecutionInfo.setNumberOfBlockletToScan(numberOfBlockletToScan);
    blockExecutionInfo.setProjectionDimensions(projectDimensions.toArray(new ProjectionDimension[projectDimensions.size()]));
    // get measures present in the current block
    List<ProjectionMeasure> currentBlockQueryMeasures = getCurrentBlockQueryMeasures(blockExecutionInfo, queryModel, blockIndex);
    blockExecutionInfo.setProjectionMeasures(currentBlockQueryMeasures.toArray(new ProjectionMeasure[currentBlockQueryMeasures.size()]));
    blockExecutionInfo.setDataBlock(blockIndex);
    // setting whether raw record query or not
    blockExecutionInfo.setRawRecordDetailQuery(queryModel.isForcedDetailRawQuery());
    // total number dimension
    blockExecutionInfo.setTotalNumberDimensionToRead(segmentProperties.getDimensionOrdinalToChunkMapping().size());
    blockExecutionInfo.setPrefetchBlocklet(!queryModel.isReadPageByPage());
    blockExecutionInfo.setTotalNumberOfMeasureToRead(segmentProperties.getMeasuresOrdinalToChunkMapping().size());
    blockExecutionInfo.setComplexDimensionInfoMap(QueryUtil.getComplexDimensionsMap(projectDimensions, segmentProperties.getDimensionOrdinalToChunkMapping(), segmentProperties.getEachComplexDimColumnValueSize(), queryProperties.columnToDictionaryMapping, queryProperties.complexFilterDimension));
    IndexKey startIndexKey = null;
    IndexKey endIndexKey = null;
    if (null != queryModel.getFilterExpressionResolverTree()) {
        // loading the filter executor tree for filter evaluation
        blockExecutionInfo.setFilterExecuterTree(FilterUtil.getFilterExecuterTree(queryModel.getFilterExpressionResolverTree(), segmentProperties, blockExecutionInfo.getComlexDimensionInfoMap()));
    }
    try {
        startIndexKey = FilterUtil.prepareDefaultStartIndexKey(segmentProperties);
        endIndexKey = FilterUtil.prepareDefaultEndIndexKey(segmentProperties);
    } catch (KeyGenException e) {
        throw new QueryExecutionException(e);
    }
    // setting the start index key of the block node
    blockExecutionInfo.setStartKey(startIndexKey);
    // setting the end index key of the block node
    blockExecutionInfo.setEndKey(endIndexKey);
    // expression dimensions
    List<CarbonDimension> expressionDimensions = new ArrayList<CarbonDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    // expression measure
    List<CarbonMeasure> expressionMeasures = new ArrayList<CarbonMeasure>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    // setting all the dimension chunk indexes to be read from file
    int numberOfElementToConsider = 0;
    // list of dimensions to be projected
    Set<Integer> allProjectionListDimensionIdexes = new LinkedHashSet<>();
    // create a list of filter dimensions present in the current block
    Set<CarbonDimension> currentBlockFilterDimensions = getCurrentBlockFilterDimensions(queryProperties.complexFilterDimension, segmentProperties);
    int[] dimensionChunkIndexes = QueryUtil.getDimensionChunkIndexes(projectDimensions, segmentProperties.getDimensionOrdinalToChunkMapping(), expressionDimensions, currentBlockFilterDimensions, allProjectionListDimensionIdexes);
    int numberOfColumnToBeReadInOneIO = Integer.parseInt(CarbonProperties.getInstance().getProperty(CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO, CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE));
    if (dimensionChunkIndexes.length > 0) {
        numberOfElementToConsider = dimensionChunkIndexes[dimensionChunkIndexes.length - 1] == segmentProperties.getBlockTodimensionOrdinalMapping().size() - 1 ? dimensionChunkIndexes.length - 1 : dimensionChunkIndexes.length;
        blockExecutionInfo.setAllSelectedDimensionColumnIndexRange(CarbonUtil.getRangeIndex(dimensionChunkIndexes, numberOfElementToConsider, numberOfColumnToBeReadInOneIO));
    } else {
        blockExecutionInfo.setAllSelectedDimensionColumnIndexRange(new int[0][0]);
    }
    // get the list of updated filter measures present in the current block
    Set<CarbonMeasure> filterMeasures = getCurrentBlockFilterMeasures(queryProperties.filterMeasures, segmentProperties);
    // list of measures to be projected
    List<Integer> allProjectionListMeasureIndexes = new ArrayList<>();
    int[] measureChunkIndexes = QueryUtil.getMeasureChunkIndexes(currentBlockQueryMeasures, expressionMeasures, segmentProperties.getMeasuresOrdinalToChunkMapping(), filterMeasures, allProjectionListMeasureIndexes);
    if (measureChunkIndexes.length > 0) {
        numberOfElementToConsider = measureChunkIndexes[measureChunkIndexes.length - 1] == segmentProperties.getMeasures().size() - 1 ? measureChunkIndexes.length - 1 : measureChunkIndexes.length;
        // setting all the measure chunk indexes to be read from file
        blockExecutionInfo.setAllSelectedMeasureIndexRange(CarbonUtil.getRangeIndex(measureChunkIndexes, numberOfElementToConsider, numberOfColumnToBeReadInOneIO));
    } else {
        blockExecutionInfo.setAllSelectedMeasureIndexRange(new int[0][0]);
    }
    // setting the indexes of list of dimension in projection list
    blockExecutionInfo.setProjectionListDimensionIndexes(ArrayUtils.toPrimitive(allProjectionListDimensionIdexes.toArray(new Integer[allProjectionListDimensionIdexes.size()])));
    // setting the indexes of list of measures in projection list
    blockExecutionInfo.setProjectionListMeasureIndexes(ArrayUtils.toPrimitive(allProjectionListMeasureIndexes.toArray(new Integer[allProjectionListMeasureIndexes.size()])));
    // setting the size of fixed key column (dictionary column)
    blockExecutionInfo.setFixedLengthKeySize(getKeySize(projectDimensions, segmentProperties));
    Set<Integer> dictionaryColumnChunkIndex = new HashSet<Integer>();
    List<Integer> noDictionaryColumnChunkIndex = new ArrayList<Integer>();
    // get the block index to be read from file for query dimension
    // for both dictionary columns and no dictionary columns
    QueryUtil.fillQueryDimensionChunkIndexes(projectDimensions, segmentProperties.getDimensionOrdinalToChunkMapping(), dictionaryColumnChunkIndex, noDictionaryColumnChunkIndex);
    int[] queryDictionaryColumnChunkIndexes = ArrayUtils.toPrimitive(dictionaryColumnChunkIndex.toArray(new Integer[dictionaryColumnChunkIndex.size()]));
    // need to sort the dictionary column as for all dimension
    // column key will be filled based on key order
    Arrays.sort(queryDictionaryColumnChunkIndexes);
    blockExecutionInfo.setDictionaryColumnChunkIndex(queryDictionaryColumnChunkIndexes);
    // setting the no dictionary column block indexes
    blockExecutionInfo.setNoDictionaryColumnChunkIndexes(ArrayUtils.toPrimitive(noDictionaryColumnChunkIndex.toArray(new Integer[noDictionaryColumnChunkIndex.size()])));
    // setting each column value size
    blockExecutionInfo.setEachColumnValueSize(segmentProperties.getEachDimColumnValueSize());
    blockExecutionInfo.setComplexColumnParentBlockIndexes(getComplexDimensionParentBlockIndexes(projectDimensions));
    blockExecutionInfo.setVectorBatchCollector(queryModel.isVectorReader());
    try {
        // to set column group and its key structure info which will be used
        // to
        // for getting the column group column data in case of final row
        // and in case of dimension aggregation
        blockExecutionInfo.setColumnGroupToKeyStructureInfo(QueryUtil.getColumnGroupKeyStructureInfo(projectDimensions, segmentProperties));
    } catch (KeyGenException e) {
        throw new QueryExecutionException(e);
    }
    // set actual query dimensions and measures. It may differ in case of restructure scenarios
    blockExecutionInfo.setActualQueryDimensions(queryModel.getProjectionDimensions().toArray(new ProjectionDimension[queryModel.getProjectionDimensions().size()]));
    blockExecutionInfo.setActualQueryMeasures(queryModel.getProjectionMeasures().toArray(new ProjectionMeasure[queryModel.getProjectionMeasures().size()]));
    DataTypeUtil.setDataTypeConverter(queryModel.getConverter());
    return blockExecutionInfo;
}

Example 15 with ProjectionDimension

use of org.apache.carbondata.core.scan.model.ProjectionDimension in project carbondata by apache.

the class PrestoCarbonVectorizedRecordReader method initBatch.

/**
 * Returns the ColumnarBatch object that will be used for all rows returned by this reader.
 * This object is reused. Calling this enables the vectorized reader. This should be called
 * before any calls to nextKeyValue/nextBatch.
 */
private void initBatch() {
    List<ProjectionDimension> queryDimension = queryModel.getProjectionDimensions();
    List<ProjectionMeasure> queryMeasures = queryModel.getProjectionMeasures();
    StructField[] fields = new StructField[queryDimension.size() + queryMeasures.size()];
    for (int i = 0; i < queryDimension.size(); i++) {
        ProjectionDimension dim = queryDimension.get(i);
        if (dim.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
            DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(dim.getDimension().getDataType());
            fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), generator.getReturnType());
        } else if (!dim.getDimension().hasEncoding(Encoding.DICTIONARY)) {
            fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), dim.getDimension().getDataType());
        } else if (dim.getDimension().isComplex()) {
            fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), dim.getDimension().getDataType());
        } else {
            fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), DataTypes.INT);
        }
    }
    for (ProjectionMeasure msr : queryMeasures) {
        DataType dataType = msr.getMeasure().getDataType();
        if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.SHORT || dataType == DataTypes.INT || dataType == DataTypes.LONG) {
            fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), msr.getMeasure().getDataType());
        } else if (DataTypes.isDecimal(dataType)) {
            fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), msr.getMeasure().getDataType());
        } else {
            fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), DataTypes.DOUBLE);
        }
    }
    columnarBatch = CarbonVectorBatch.allocate(fields);
    CarbonColumnVector[] vectors = new CarbonColumnVector[fields.length];
    boolean[] filteredRows = new boolean[columnarBatch.capacity()];
    for (int i = 0; i < fields.length; i++) {
        vectors[i] = new CarbonColumnVectorWrapper(columnarBatch.column(i), filteredRows);
    }
    carbonColumnarBatch = new CarbonColumnarBatch(vectors, columnarBatch.capacity(), filteredRows);
}