Examples with CarbonDimension org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension used on opensource projects

Example 36 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class CarbonParserFactory method createParser.

/**
   * This method may be called recursively if the carbon column is complex type.
   *
   * @param carbonColumn
   * @param complexDelimiters, these delimiters which are used to separate the complex data types.
   * @param depth              It is like depth of tree, if column has children then depth is 1,
   *                           And depth becomes 2 if children has children.
   *                           This depth is used select the complex
   *                           delimiters
   * @return GenericParser
   */
private static GenericParser createParser(CarbonColumn carbonColumn, String[] complexDelimiters, String nullFormat, int depth) {
    switch(carbonColumn.getDataType()) {
        case ARRAY:
            List<CarbonDimension> listOfChildDimensions = ((CarbonDimension) carbonColumn).getListOfChildDimensions();
            // Create array parser with complex delimiter
            ArrayParserImpl arrayParser = new ArrayParserImpl(complexDelimiters[depth], nullFormat);
            for (CarbonDimension dimension : listOfChildDimensions) {
                arrayParser.addChildren(createParser(dimension, complexDelimiters, nullFormat, depth + 1));
            }
            return arrayParser;
        case STRUCT:
            List<CarbonDimension> dimensions = ((CarbonDimension) carbonColumn).getListOfChildDimensions();
            // Create struct parser with complex delimiter
            StructParserImpl parser = new StructParserImpl(complexDelimiters[depth], nullFormat);
            for (CarbonDimension dimension : dimensions) {
                parser.addChildren(createParser(dimension, complexDelimiters, nullFormat, depth + 1));
            }
            return parser;
        case MAP:
            throw new UnsupportedOperationException("Complex type Map is not supported yet");
        default:
            return new PrimitiveParserImpl();
    }
}

Example 37 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class CarbonCompactionExecutor method prepareQueryModel.

/**
   * Preparing of the query model.
   *
   * @param blockList
   * @return
   */
private QueryModel prepareQueryModel(List<TableBlockInfo> blockList) {
    QueryModel model = new QueryModel();
    model.setTableBlockInfos(blockList);
    model.setForcedDetailRawQuery(true);
    model.setFilterExpressionResolverTree(null);
    List<QueryDimension> dims = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(carbonTable.getFactTableName());
    for (CarbonDimension dim : dimensions) {
        // check if dimension is deleted
        QueryDimension queryDimension = new QueryDimension(dim.getColName());
        queryDimension.setDimension(dim);
        dims.add(queryDimension);
    }
    model.setQueryDimension(dims);
    List<QueryMeasure> msrs = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(carbonTable.getFactTableName());
    for (CarbonMeasure carbonMeasure : measures) {
        // check if measure is deleted
        QueryMeasure queryMeasure = new QueryMeasure(carbonMeasure.getColName());
        queryMeasure.setMeasure(carbonMeasure);
        msrs.add(queryMeasure);
    }
    model.setQueryMeasures(msrs);
    model.setQueryId(System.nanoTime() + "");
    model.setAbsoluteTableIdentifier(carbonTable.getAbsoluteTableIdentifier());
    model.setTable(carbonTable);
    return model;
}

Example 38 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class CompactionResultSortProcessor method initSortDataRows.

/**
   * create an instance of sort data rows
   */
private void initSortDataRows() throws Exception {
    measureCount = carbonTable.getMeasureByTableName(tableName).size();
    List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(tableName);
    noDictionaryColMapping = new boolean[dimensions.size()];
    int i = 0;
    for (CarbonDimension dimension : dimensions) {
        if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
            i++;
            continue;
        }
        noDictionaryColMapping[i++] = true;
        noDictionaryCount++;
    }
    dimensionColumnCount = dimensions.size();
    SortParameters parameters = createSortParameters();
    intermediateFileMerger = new SortIntermediateFileMerger(parameters);
    // TODO: Now it is only supported onheap merge, but we can have unsafe merge
    // as well by using UnsafeSortDataRows.
    this.sortDataRows = new SortDataRows(parameters, intermediateFileMerger);
    try {
        this.sortDataRows.initialize();
    } catch (CarbonSortKeyAndGroupByException e) {
        LOGGER.error(e);
        throw new Exception("Error initializing sort data rows object during compaction: " + e.getMessage());
    }
}

Example 39 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class StoreCreator method writeDictionary.

private static void writeDictionary(String factFilePath, CarbonTable table) throws Exception {
    BufferedReader reader = new BufferedReader(new FileReader(factFilePath));
    String header = reader.readLine();
    String[] split = header.split(",");
    List<CarbonColumn> allCols = new ArrayList<CarbonColumn>();
    List<CarbonDimension> dims = table.getDimensionByTableName(table.getFactTableName());
    allCols.addAll(dims);
    List<CarbonMeasure> msrs = table.getMeasureByTableName(table.getFactTableName());
    allCols.addAll(msrs);
    Set<String>[] set = new HashSet[dims.size()];
    for (int i = 0; i < set.length; i++) {
        set[i] = new HashSet<String>();
    }
    String line = reader.readLine();
    while (line != null) {
        String[] data = line.split(",");
        for (int i = 0; i < set.length; i++) {
            set[i].add(data[i]);
        }
        line = reader.readLine();
    }
    Cache dictCache = CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY, absoluteTableIdentifier.getStorePath());
    for (int i = 0; i < set.length; i++) {
        ColumnIdentifier columnIdentifier = new ColumnIdentifier(dims.get(i).getColumnId(), null, null);
        CarbonDictionaryWriter writer = new CarbonDictionaryWriterImpl(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier(), columnIdentifier);
        for (String value : set[i]) {
            writer.write(value);
        }
        writer.close();
        writer.commit();
        Dictionary dict = (Dictionary) dictCache.get(new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier.getCarbonTableIdentifier(), columnIdentifier, dims.get(i).getDataType()));
        CarbonDictionarySortInfoPreparator preparator = new CarbonDictionarySortInfoPreparator();
        List<String> newDistinctValues = new ArrayList<String>();
        CarbonDictionarySortInfo dictionarySortInfo = preparator.getDictionarySortInfo(newDistinctValues, dict, dims.get(i).getDataType());
        CarbonDictionarySortIndexWriter carbonDictionaryWriter = new CarbonDictionarySortIndexWriterImpl(absoluteTableIdentifier.getCarbonTableIdentifier(), columnIdentifier, absoluteTableIdentifier.getStorePath());
        try {
            carbonDictionaryWriter.writeSortIndex(dictionarySortInfo.getSortIndex());
            carbonDictionaryWriter.writeInvertedSortIndex(dictionarySortInfo.getSortIndexInverted());
        } finally {
            carbonDictionaryWriter.close();
        }
    }
    reader.close();
}

Example 40 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class CarbonInputFormatUtil method processFilterExpression.

public static void processFilterExpression(Expression filterExpression, CarbonTable carbonTable) {
    List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(carbonTable.getFactTableName());
    List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(carbonTable.getFactTableName());
    QueryModel.processFilterExpression(filterExpression, dimensions, measures);
    if (null != filterExpression) {
        // Optimize Filter Expression and fit RANGE filters is conditions apply.
        FilterOptimizer rangeFilterOptimizer = new RangeFilterOptmizer(new FilterOptimizerBasic(), filterExpression);
        rangeFilterOptimizer.optimizeFilter();
    }
}