Examples with CarbonDimension org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension used on opensource projects

Example 51 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class SegmentProperties method assignComplexOrdinal.

/**
   * Read all primitive/complex children and set it as list of child carbon dimension to parent
   * dimension
   */
private int assignComplexOrdinal(CarbonDimension parentDimension, int complexDimensionOrdinal) {
    for (int i = 0; i < parentDimension.getNumberOfChild(); i++) {
        CarbonDimension dimension = parentDimension.getListOfChildDimensions().get(i);
        if (dimension.getNumberOfChild() > 0) {
            dimension.setComplexTypeOridnal(++complexDimensionOrdinal);
            complexDimensionOrdinal = assignComplexOrdinal(dimension, complexDimensionOrdinal);
        } else {
            parentDimension.getListOfChildDimensions().get(i).setComplexTypeOridnal(++complexDimensionOrdinal);
        }
    }
    return complexDimensionOrdinal;
}

Example 52 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class SegmentProperties method readAllComplexTypeChildren.

/**
   * Read all primitive/complex children and set it as list of child carbon dimension to parent
   * dimension
   *
   * @param dimensionOrdinal
   * @param childCount
   * @param listOfColumns
   * @param parentDimension
   * @return
   */
private int readAllComplexTypeChildren(int dimensionOrdinal, int childCount, List<ColumnSchema> listOfColumns, CarbonDimension parentDimension, int complexDimensionOrdinal) {
    for (int i = 0; i < childCount; i++) {
        ColumnSchema columnSchema = listOfColumns.get(dimensionOrdinal);
        if (columnSchema.isDimensionColumn()) {
            if (columnSchema.getNumberOfChild() > 0) {
                CarbonDimension complexDimension = new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, complexDimensionOrdinal++);
                complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
                parentDimension.getListOfChildDimensions().add(complexDimension);
                dimensionOrdinal = readAllComplexTypeChildren(dimensionOrdinal, columnSchema.getNumberOfChild(), listOfColumns, complexDimension, complexDimensionOrdinal);
            } else {
                parentDimension.getListOfChildDimensions().add(new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, complexDimensionOrdinal++));
            }
        }
    }
    return dimensionOrdinal;
}

Example 53 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class CarbonFactDataHandlerModel method getCarbonFactDataHandlerModel.

/**
   * This method will create a model object for carbon fact data handler
   *
   * @param loadModel
   * @return
   */
public static CarbonFactDataHandlerModel getCarbonFactDataHandlerModel(CarbonLoadModel loadModel, CarbonTable carbonTable, SegmentProperties segmentProperties, String tableName, String tempStoreLocation) {
    CarbonFactDataHandlerModel carbonFactDataHandlerModel = new CarbonFactDataHandlerModel();
    carbonFactDataHandlerModel.setSchemaUpdatedTimeStamp(carbonTable.getTableLastUpdatedTime());
    carbonFactDataHandlerModel.setDatabaseName(loadModel.getDatabaseName());
    carbonFactDataHandlerModel.setTableName(tableName);
    carbonFactDataHandlerModel.setMeasureCount(segmentProperties.getMeasures().size());
    carbonFactDataHandlerModel.setStoreLocation(tempStoreLocation);
    carbonFactDataHandlerModel.setDimLens(segmentProperties.getDimColumnsCardinality());
    carbonFactDataHandlerModel.setSegmentProperties(segmentProperties);
    carbonFactDataHandlerModel.setNoDictionaryCount(segmentProperties.getNumberOfNoDictionaryDimension());
    carbonFactDataHandlerModel.setDimensionCount(segmentProperties.getDimensions().size() - carbonFactDataHandlerModel.getNoDictionaryCount());
    List<ColumnSchema> wrapperColumnSchema = CarbonUtil.getColumnSchemaList(carbonTable.getDimensionByTableName(tableName), carbonTable.getMeasureByTableName(tableName));
    carbonFactDataHandlerModel.setWrapperColumnSchema(wrapperColumnSchema);
    // get the cardinality for all all the columns including no dictionary columns
    int[] formattedCardinality = CarbonUtil.getFormattedCardinality(segmentProperties.getDimColumnsCardinality(), wrapperColumnSchema);
    carbonFactDataHandlerModel.setColCardinality(formattedCardinality);
    //TO-DO Need to handle complex types here .
    Map<Integer, GenericDataType> complexIndexMap = new HashMap<Integer, GenericDataType>(segmentProperties.getComplexDimensions().size());
    carbonFactDataHandlerModel.setComplexIndexMap(complexIndexMap);
    DataType[] aggType = new DataType[segmentProperties.getMeasures().size()];
    int i = 0;
    for (CarbonMeasure msr : segmentProperties.getMeasures()) {
        aggType[i++] = msr.getDataType();
    }
    carbonFactDataHandlerModel.setMeasureDataType(aggType);
    String carbonDataDirectoryPath = CarbonDataProcessorUtil.checkAndCreateCarbonStoreLocation(loadModel.getStorePath(), loadModel.getDatabaseName(), tableName, loadModel.getPartitionId(), loadModel.getSegmentId());
    carbonFactDataHandlerModel.setCarbonDataDirectoryPath(carbonDataDirectoryPath);
    List<CarbonDimension> dimensionByTableName = loadModel.getCarbonDataLoadSchema().getCarbonTable().getDimensionByTableName(tableName);
    boolean[] isUseInvertedIndexes = new boolean[dimensionByTableName.size()];
    int index = 0;
    for (CarbonDimension dimension : dimensionByTableName) {
        isUseInvertedIndexes[index++] = dimension.isUseInvertedIndex();
    }
    carbonFactDataHandlerModel.setIsUseInvertedIndex(isUseInvertedIndexes);
    carbonFactDataHandlerModel.setPrimitiveDimLens(segmentProperties.getDimColumnsCardinality());
    carbonFactDataHandlerModel.setBlockSizeInMB(carbonTable.getBlockSizeInMB());
    return carbonFactDataHandlerModel;
}

Example 54 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class QueryModel method setDimAndMsrColumnNode.

private static void setDimAndMsrColumnNode(List<CarbonDimension> dimensions, List<CarbonMeasure> measures, ColumnExpression col) {
    CarbonDimension dim;
    CarbonMeasure msr;
    String columnName;
    columnName = col.getColumnName();
    dim = CarbonUtil.findDimension(dimensions, columnName);
    col.setCarbonColumn(dim);
    col.setDimension(dim);
    col.setDimension(true);
    if (null == dim) {
        msr = getCarbonMetadataMeasure(columnName, measures);
        col.setCarbonColumn(msr);
        col.setDimension(false);
    }
}

Example 55 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class ConditionalFilterResolverImpl method resolve.

/**
   * This API will resolve the filter expression and generates the
   * dictionaries for executing/evaluating the filter expressions in the
   * executer layer.
   *
   * @throws FilterUnsupportedException
   */
@Override
public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier) throws FilterUnsupportedException, IOException {
    FilterResolverMetadata metadata = new FilterResolverMetadata();
    metadata.setTableIdentifier(absoluteTableIdentifier);
    if ((!isExpressionResolve) && exp instanceof BinaryConditionalExpression) {
        BinaryConditionalExpression binaryConditionalExpression = (BinaryConditionalExpression) exp;
        Expression leftExp = binaryConditionalExpression.getLeft();
        Expression rightExp = binaryConditionalExpression.getRight();
        if (leftExp instanceof ColumnExpression) {
            ColumnExpression columnExpression = (ColumnExpression) leftExp;
            metadata.setColumnExpression(columnExpression);
            metadata.setExpression(rightExp);
            metadata.setIncludeFilter(isIncludeFilter);
            // If imei=imei comes in filter condition then we need to
            // skip processing of right expression.
            // This flow has reached here assuming that this is a single
            // column expression.
            // we need to check if the other expression contains column
            // expression or not in depth.
            CarbonDimension dimension = columnExpression.getDimension();
            if (FilterUtil.checkIfExpressionContainsColumn(rightExp) || FilterUtil.isExpressionNeedsToResolved(rightExp, isIncludeFilter) && dimension.hasEncoding(Encoding.DICTIONARY) && !dimension.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
                isExpressionResolve = true;
            } else {
                //Visitor pattern is been used in this scenario inorder to populate the
                // dimColResolvedFilterInfo
                //visitable object with filter member values based on the visitor type, currently there
                //3 types of visitors custom,direct and no dictionary, all types of visitor populate
                //the visitable instance as per its buisness logic which is different for all the
                // visitors.
                dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression, exp), metadata);
            }
        } else if (rightExp instanceof ColumnExpression) {
            ColumnExpression columnExpression = (ColumnExpression) rightExp;
            metadata.setColumnExpression(columnExpression);
            metadata.setExpression(leftExp);
            metadata.setIncludeFilter(isIncludeFilter);
            if (columnExpression.getDataType().equals(DataType.TIMESTAMP) || columnExpression.getDataType().equals(DataType.DATE)) {
                isExpressionResolve = true;
            } else {
                // expression or not in depth.
                if (FilterUtil.checkIfExpressionContainsColumn(leftExp)) {
                    isExpressionResolve = true;
                } else {
                    dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression, exp), metadata);
                }
            }
        } else {
            isExpressionResolve = true;
        }
    }
    if (isExpressionResolve && exp instanceof ConditionalExpression) {
        ConditionalExpression conditionalExpression = (ConditionalExpression) exp;
        List<ColumnExpression> columnList = conditionalExpression.getColumnList();
        metadata.setColumnExpression(columnList.get(0));
        metadata.setExpression(exp);
        metadata.setIncludeFilter(isIncludeFilter);
        if (!columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) || columnList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY) || (exp instanceof RangeExpression)) {
            dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnList.get(0), exp), metadata);
        } else if (columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) && !(columnList.get(0).getDimension().getDataType() == org.apache.carbondata.core.metadata.datatype.DataType.STRUCT || columnList.get(0).getDimension().getDataType() == org.apache.carbondata.core.metadata.datatype.DataType.ARRAY)) {
            dimColResolvedFilterInfo.setFilterValues(FilterUtil.getFilterListForAllValues(absoluteTableIdentifier, exp, columnList.get(0), isIncludeFilter));
            dimColResolvedFilterInfo.setColumnIndex(columnList.get(0).getDimension().getOrdinal());
            dimColResolvedFilterInfo.setDimension(columnList.get(0).getDimension());
        }
    }
}