Examples with CarbonMeasure org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure used on opensource projects

Example 11 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class CarbonCompactionExecutor method prepareQueryModel.

/**
   * Preparing of the query model.
   *
   * @param blockList
   * @return
   */
private QueryModel prepareQueryModel(List<TableBlockInfo> blockList) {
    QueryModel model = new QueryModel();
    model.setTableBlockInfos(blockList);
    model.setForcedDetailRawQuery(true);
    model.setFilterExpressionResolverTree(null);
    List<QueryDimension> dims = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(carbonTable.getFactTableName());
    for (CarbonDimension dim : dimensions) {
        // check if dimension is deleted
        QueryDimension queryDimension = new QueryDimension(dim.getColName());
        queryDimension.setDimension(dim);
        dims.add(queryDimension);
    }
    model.setQueryDimension(dims);
    List<QueryMeasure> msrs = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(carbonTable.getFactTableName());
    for (CarbonMeasure carbonMeasure : measures) {
        // check if measure is deleted
        QueryMeasure queryMeasure = new QueryMeasure(carbonMeasure.getColName());
        queryMeasure.setMeasure(carbonMeasure);
        msrs.add(queryMeasure);
    }
    model.setQueryMeasures(msrs);
    model.setQueryId(System.nanoTime() + "");
    model.setAbsoluteTableIdentifier(carbonTable.getAbsoluteTableIdentifier());
    model.setTable(carbonTable);
    return model;
}

Example 12 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class StoreCreator method writeDictionary.

private static void writeDictionary(String factFilePath, CarbonTable table) throws Exception {
    BufferedReader reader = new BufferedReader(new FileReader(factFilePath));
    String header = reader.readLine();
    String[] split = header.split(",");
    List<CarbonColumn> allCols = new ArrayList<CarbonColumn>();
    List<CarbonDimension> dims = table.getDimensionByTableName(table.getFactTableName());
    allCols.addAll(dims);
    List<CarbonMeasure> msrs = table.getMeasureByTableName(table.getFactTableName());
    allCols.addAll(msrs);
    Set<String>[] set = new HashSet[dims.size()];
    for (int i = 0; i < set.length; i++) {
        set[i] = new HashSet<String>();
    }
    String line = reader.readLine();
    while (line != null) {
        String[] data = line.split(",");
        for (int i = 0; i < set.length; i++) {
            set[i].add(data[i]);
        }
        line = reader.readLine();
    }
    Cache dictCache = CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY, absoluteTableIdentifier.getStorePath());
    for (int i = 0; i < set.length; i++) {
        ColumnIdentifier columnIdentifier = new ColumnIdentifier(dims.get(i).getColumnId(), null, null);
        CarbonDictionaryWriter writer = new CarbonDictionaryWriterImpl(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier(), columnIdentifier);
        for (String value : set[i]) {
            writer.write(value);
        }
        writer.close();
        writer.commit();
        Dictionary dict = (Dictionary) dictCache.get(new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier.getCarbonTableIdentifier(), columnIdentifier, dims.get(i).getDataType()));
        CarbonDictionarySortInfoPreparator preparator = new CarbonDictionarySortInfoPreparator();
        List<String> newDistinctValues = new ArrayList<String>();
        CarbonDictionarySortInfo dictionarySortInfo = preparator.getDictionarySortInfo(newDistinctValues, dict, dims.get(i).getDataType());
        CarbonDictionarySortIndexWriter carbonDictionaryWriter = new CarbonDictionarySortIndexWriterImpl(absoluteTableIdentifier.getCarbonTableIdentifier(), columnIdentifier, absoluteTableIdentifier.getStorePath());
        try {
            carbonDictionaryWriter.writeSortIndex(dictionarySortInfo.getSortIndex());
            carbonDictionaryWriter.writeInvertedSortIndex(dictionarySortInfo.getSortIndexInverted());
        } finally {
            carbonDictionaryWriter.close();
        }
    }
    reader.close();
}

Example 13 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class CarbonInputFormatUtil method processFilterExpression.

public static void processFilterExpression(Expression filterExpression, CarbonTable carbonTable) {
    List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(carbonTable.getFactTableName());
    List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(carbonTable.getFactTableName());
    QueryModel.processFilterExpression(filterExpression, dimensions, measures);
    if (null != filterExpression) {
        // Optimize Filter Expression and fit RANGE filters is conditions apply.
        FilterOptimizer rangeFilterOptimizer = new RangeFilterOptmizer(new FilterOptimizerBasic(), filterExpression);
        rangeFilterOptimizer.optimizeFilter();
    }
}

Example 14 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class CarbonInputFormatUtil method createQueryPlan.

public static CarbonQueryPlan createQueryPlan(CarbonTable carbonTable, String columnString) {
    String[] columns = null;
    if (columnString != null) {
        columns = columnString.split(",");
    }
    String factTableName = carbonTable.getFactTableName();
    CarbonQueryPlan plan = new CarbonQueryPlan(carbonTable.getDatabaseName(), factTableName);
    // fill dimensions
    // If columns are null, set all dimensions and measures
    int i = 0;
    if (columns != null) {
        for (String column : columns) {
            CarbonDimension dimensionByName = carbonTable.getDimensionByName(factTableName, column);
            if (dimensionByName != null) {
                addQueryDimension(plan, i, dimensionByName);
                i++;
            } else {
                CarbonMeasure measure = carbonTable.getMeasureByName(factTableName, column);
                if (measure == null) {
                    throw new RuntimeException(column + " column not found in the table " + factTableName);
                }
                addQueryMeasure(plan, i, measure);
                i++;
            }
        }
    }
    plan.setQueryId(System.nanoTime() + "");
    return plan;
}

Example 15 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class CarbonUtil method getColumnSchemaList.

public static List<ColumnSchema> getColumnSchemaList(List<CarbonDimension> carbonDimensionsList, List<CarbonMeasure> carbonMeasureList) {
    List<ColumnSchema> wrapperColumnSchemaList = new ArrayList<ColumnSchema>();
    fillCollumnSchemaListForComplexDims(carbonDimensionsList, wrapperColumnSchemaList);
    for (CarbonMeasure carbonMeasure : carbonMeasureList) {
        wrapperColumnSchemaList.add(carbonMeasure.getColumnSchema());
    }
    return wrapperColumnSchemaList;
}