Examples with ColumnSchema org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema used on opensource projects

Example 16 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class TableSchemaBuilder method addColumn.

public TableSchemaBuilder addColumn(StructField field, boolean isSortColumn) {
    Objects.requireNonNull(field);
    checkRepeatColumnName(field);
    ColumnSchema newColumn = new ColumnSchema();
    newColumn.setColumnName(field.getFieldName());
    newColumn.setDataType(field.getDataType());
    if (isSortColumn || field.getDataType() == DataTypes.STRING || field.getDataType() == DataTypes.DATE || field.getDataType() == DataTypes.TIMESTAMP) {
        newColumn.setDimensionColumn(true);
    } else {
        newColumn.setDimensionColumn(false);
    }
    newColumn.setSchemaOrdinal(ordinal++);
    newColumn.setColumnar(true);
    newColumn.setColumnUniqueId(UUID.randomUUID().toString());
    newColumn.setColumnReferenceId(newColumn.getColumnUniqueId());
    newColumn.setEncodingList(createEncoding(field.getDataType(), isSortColumn));
    if (isSortColumn) {
        sortColumns.add(newColumn);
        newColumn.setSortColumn(true);
    } else {
        otherColumns.add(newColumn);
    }
    return this;
}

Example 17 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class TableSchemaBuilder method build.

public TableSchema build() {
    TableSchema schema = new TableSchema();
    schema.setTableName(tableName);
    schema.setTableId(UUID.randomUUID().toString());
    schema.setPartitionInfo(null);
    schema.setBucketingInfo(null);
    SchemaEvolution schemaEvol = new SchemaEvolution();
    schemaEvol.setSchemaEvolutionEntryList(new ArrayList<SchemaEvolutionEntry>());
    schema.setSchemaEvalution(schemaEvol);
    List<ColumnSchema> allColumns = new LinkedList<>(sortColumns);
    allColumns.addAll(otherColumns);
    schema.setListOfColumns(allColumns);
    if (blockSize > 0) {
        Map<String, String> property = new HashMap<>();
        property.put(CarbonCommonConstants.TABLE_BLOCKSIZE, String.valueOf(blockSize));
        schema.setTableProperties(property);
    }
    return schema;
}

Example 18 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class CarbonStreamRecordReader method initializeFilter.

private void initializeFilter() {
    List<ColumnSchema> wrapperColumnSchemaList = CarbonUtil.getColumnSchemaList(carbonTable.getDimensionByTableName(carbonTable.getTableName()), carbonTable.getMeasureByTableName(carbonTable.getTableName()));
    int[] dimLensWithComplex = new int[wrapperColumnSchemaList.size()];
    for (int i = 0; i < dimLensWithComplex.length; i++) {
        dimLensWithComplex[i] = Integer.MAX_VALUE;
    }
    int[] dictionaryColumnCardinality = CarbonUtil.getFormattedCardinality(dimLensWithComplex, wrapperColumnSchemaList);
    SegmentProperties segmentProperties = new SegmentProperties(wrapperColumnSchemaList, dictionaryColumnCardinality);
    Map<Integer, GenericQueryType> complexDimensionInfoMap = new HashMap<>();
    FilterResolverIntf resolverIntf = model.getFilterExpressionResolverTree();
    filter = FilterUtil.getFilterExecuterTree(resolverIntf, segmentProperties, complexDimensionInfoMap);
    // for row filter, we need update column index
    FilterUtil.updateIndexOfColumnExpression(resolverIntf.getFilterExpression(), carbonTable.getDimensionOrdinalMax());
}

Example 19 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class SegmentTaskIndexStore method loadBlocks.

/**
 * Below method will be used to load the blocks
 *
 * @param tableBlockInfoList
 * @return loaded segment
 * @throws IOException
 */
private AbstractIndex loadBlocks(TaskBucketHolder taskBucketHolder, List<TableBlockInfo> tableBlockInfoList, AbsoluteTableIdentifier tableIdentifier) throws IOException {
    // all the block of one task id will be loaded together
    // so creating a list which will have all the data file meta data to of one task
    List<DataFileFooter> footerList = CarbonUtil.readCarbonIndexFile(taskBucketHolder.taskNo, taskBucketHolder.bucketNumber, tableBlockInfoList, tableIdentifier);
    // Reuse SegmentProperties object if tableIdentifier, columnsInTable and columnCardinality are
    // the same.
    List<ColumnSchema> columnsInTable = footerList.get(0).getColumnInTable();
    int[] columnCardinality = footerList.get(0).getSegmentInfo().getColumnCardinality();
    SegmentPropertiesWrapper segmentPropertiesWrapper = new SegmentPropertiesWrapper(tableIdentifier, columnsInTable, columnCardinality);
    SegmentProperties segmentProperties;
    if (this.segmentProperties.containsKey(segmentPropertiesWrapper)) {
        segmentProperties = this.segmentProperties.get(segmentPropertiesWrapper);
    } else {
        // create a metadata details
        // this will be useful in query handling
        // all the data file metadata will have common segment properties we
        // can use first one to get create the segment properties
        segmentProperties = new SegmentProperties(columnsInTable, columnCardinality);
        this.segmentProperties.put(segmentPropertiesWrapper, segmentProperties);
    }
    AbstractIndex segment = new SegmentTaskIndex(segmentProperties);
    // file path of only first block is passed as it all table block info path of
    // same task id will be same
    segment.buildIndex(footerList);
    return segment;
}

Example 20 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class SegmentProperties method fillDimensionAndMeasureDetails.

/**
 * below method will fill dimension and measure detail of the block.
 *
 * @param columnsInTable
 * @param columnCardinality
 */
private void fillDimensionAndMeasureDetails(List<ColumnSchema> columnsInTable, int[] columnCardinality) {
    ColumnSchema columnSchema = null;
    // ordinal will be required to read the data from file block
    int dimensonOrdinal = 0;
    int measureOrdinal = -1;
    // table ordinal is actually a schema ordinal this is required as
    // cardinality array
    // which is stored in segment info contains -1 if that particular column
    // is n
    int tableOrdinal = -1;
    // creating a list as we do not know how many dimension not participated
    // in the mdkey
    List<Integer> cardinalityIndexForNormalDimensionColumn = new ArrayList<Integer>(columnsInTable.size());
    // creating a list as we do not know how many dimension not participated
    // in the mdkey
    List<Integer> cardinalityIndexForComplexDimensionColumn = new ArrayList<Integer>(columnsInTable.size());
    boolean isComplexDimensionStarted = false;
    CarbonDimension carbonDimension = null;
    // to store the position of dimension in surrogate key array which is
    // participating in mdkey
    int keyOrdinal = 0;
    int previousColumnGroup = -1;
    // to store the ordinal of the column group ordinal
    int columnGroupOrdinal = 0;
    int counter = 0;
    int complexTypeOrdinal = -1;
    while (counter < columnsInTable.size()) {
        columnSchema = columnsInTable.get(counter);
        if (columnSchema.isDimensionColumn()) {
            tableOrdinal++;
            // column as it was not the part of mdkey
            if (CarbonUtil.hasEncoding(columnSchema.getEncodingList(), Encoding.DICTIONARY) && !isComplexDimensionStarted && columnSchema.getNumberOfChild() == 0) {
                cardinalityIndexForNormalDimensionColumn.add(tableOrdinal);
                if (columnSchema.isSortColumn()) {
                    this.numberOfSortColumns++;
                }
                if (columnSchema.isColumnar()) {
                    // if it is a columnar dimension participated in mdkey then added
                    // key ordinal and dimension ordinal
                    carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, keyOrdinal++, -1, -1);
                } else {
                    // in this case ordinal of the column group will be 0
                    if (previousColumnGroup != columnSchema.getColumnGroupId()) {
                        columnGroupOrdinal = 0;
                        carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, keyOrdinal++, columnGroupOrdinal++, -1);
                    } else // if previous dimension  column group id is same as current then
                    // then its belongs to same row group
                    {
                        carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, keyOrdinal++, columnGroupOrdinal++, -1);
                    }
                    previousColumnGroup = columnSchema.getColumnGroupId();
                }
            } else // will be added to complex type
            if (isComplexDimensionStarted || columnSchema.getDataType().isComplexType()) {
                cardinalityIndexForComplexDimensionColumn.add(tableOrdinal);
                carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, ++complexTypeOrdinal);
                carbonDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
                complexDimensions.add(carbonDimension);
                isComplexDimensionStarted = true;
                int previouseOrdinal = dimensonOrdinal;
                dimensonOrdinal = readAllComplexTypeChildren(dimensonOrdinal, columnSchema.getNumberOfChild(), columnsInTable, carbonDimension, complexTypeOrdinal);
                int numberOfChildrenDimensionAdded = dimensonOrdinal - previouseOrdinal;
                for (int i = 0; i < numberOfChildrenDimensionAdded; i++) {
                    cardinalityIndexForComplexDimensionColumn.add(++tableOrdinal);
                }
                counter = dimensonOrdinal;
                complexTypeOrdinal = assignComplexOrdinal(carbonDimension, complexTypeOrdinal);
                continue;
            } else {
                // for no dictionary dimension
                carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, -1);
                numberOfNoDictionaryDimension++;
                if (columnSchema.isSortColumn()) {
                    this.numberOfSortColumns++;
                    this.numberOfNoDictSortColumns++;
                }
            }
            dimensions.add(carbonDimension);
        } else {
            measures.add(new CarbonMeasure(columnSchema, ++measureOrdinal));
        }
        counter++;
    }
    lastDimensionColOrdinal = dimensonOrdinal;
    dimColumnsCardinality = new int[cardinalityIndexForNormalDimensionColumn.size()];
    complexDimColumnCardinality = new int[cardinalityIndexForComplexDimensionColumn.size()];
    int index = 0;
    // generator
    for (Integer cardinalityArrayIndex : cardinalityIndexForNormalDimensionColumn) {
        dimColumnsCardinality[index++] = columnCardinality[cardinalityArrayIndex];
    }
    index = 0;
    // key generator
    for (Integer cardinalityArrayIndex : cardinalityIndexForComplexDimensionColumn) {
        complexDimColumnCardinality[index++] = columnCardinality[cardinalityArrayIndex];
    }
}