Examples with CarbonDimension org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension used on opensource projects

Example 81 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class SegmentProperties method readAllComplexTypeChildren.

/**
   * Read all primitive/complex children and set it as list of child carbon dimension to parent
   * dimension
   *
   * @param dimensionOrdinal
   * @param childCount
   * @param listOfColumns
   * @param parentDimension
   * @return
   */
private int readAllComplexTypeChildren(int dimensionOrdinal, int childCount, List<ColumnSchema> listOfColumns, CarbonDimension parentDimension, int complexDimensionOrdinal) {
    for (int i = 0; i < childCount; i++) {
        ColumnSchema columnSchema = listOfColumns.get(dimensionOrdinal);
        if (columnSchema.isDimensionColumn()) {
            if (columnSchema.getNumberOfChild() > 0) {
                CarbonDimension complexDimension = new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, complexDimensionOrdinal++);
                complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
                parentDimension.getListOfChildDimensions().add(complexDimension);
                dimensionOrdinal = readAllComplexTypeChildren(dimensionOrdinal, columnSchema.getNumberOfChild(), listOfColumns, complexDimension, complexDimensionOrdinal);
            } else {
                parentDimension.getListOfChildDimensions().add(new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, complexDimensionOrdinal++));
            }
        }
    }
    return dimensionOrdinal;
}

Example 82 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class CarbonFactDataHandlerModel method getCarbonFactDataHandlerModel.

/**
   * This method will create a model object for carbon fact data handler
   *
   * @param loadModel
   * @return
   */
public static CarbonFactDataHandlerModel getCarbonFactDataHandlerModel(CarbonLoadModel loadModel, CarbonTable carbonTable, SegmentProperties segmentProperties, String tableName, String tempStoreLocation) {
    CarbonFactDataHandlerModel carbonFactDataHandlerModel = new CarbonFactDataHandlerModel();
    carbonFactDataHandlerModel.setSchemaUpdatedTimeStamp(carbonTable.getTableLastUpdatedTime());
    carbonFactDataHandlerModel.setDatabaseName(loadModel.getDatabaseName());
    carbonFactDataHandlerModel.setTableName(tableName);
    carbonFactDataHandlerModel.setMeasureCount(segmentProperties.getMeasures().size());
    carbonFactDataHandlerModel.setStoreLocation(tempStoreLocation);
    carbonFactDataHandlerModel.setDimLens(segmentProperties.getDimColumnsCardinality());
    carbonFactDataHandlerModel.setSegmentProperties(segmentProperties);
    carbonFactDataHandlerModel.setNoDictionaryCount(segmentProperties.getNumberOfNoDictionaryDimension());
    carbonFactDataHandlerModel.setDimensionCount(segmentProperties.getDimensions().size() - carbonFactDataHandlerModel.getNoDictionaryCount());
    List<ColumnSchema> wrapperColumnSchema = CarbonUtil.getColumnSchemaList(carbonTable.getDimensionByTableName(tableName), carbonTable.getMeasureByTableName(tableName));
    carbonFactDataHandlerModel.setWrapperColumnSchema(wrapperColumnSchema);
    // get the cardinality for all all the columns including no dictionary columns
    int[] formattedCardinality = CarbonUtil.getFormattedCardinality(segmentProperties.getDimColumnsCardinality(), wrapperColumnSchema);
    carbonFactDataHandlerModel.setColCardinality(formattedCardinality);
    //TO-DO Need to handle complex types here .
    Map<Integer, GenericDataType> complexIndexMap = new HashMap<Integer, GenericDataType>(segmentProperties.getComplexDimensions().size());
    carbonFactDataHandlerModel.setComplexIndexMap(complexIndexMap);
    DataType[] aggType = new DataType[segmentProperties.getMeasures().size()];
    int i = 0;
    for (CarbonMeasure msr : segmentProperties.getMeasures()) {
        aggType[i++] = msr.getDataType();
    }
    carbonFactDataHandlerModel.setMeasureDataType(aggType);
    String carbonDataDirectoryPath = CarbonDataProcessorUtil.checkAndCreateCarbonStoreLocation(loadModel.getStorePath(), loadModel.getDatabaseName(), tableName, loadModel.getPartitionId(), loadModel.getSegmentId());
    carbonFactDataHandlerModel.setCarbonDataDirectoryPath(carbonDataDirectoryPath);
    List<CarbonDimension> dimensionByTableName = loadModel.getCarbonDataLoadSchema().getCarbonTable().getDimensionByTableName(tableName);
    boolean[] isUseInvertedIndexes = new boolean[dimensionByTableName.size()];
    int index = 0;
    for (CarbonDimension dimension : dimensionByTableName) {
        isUseInvertedIndexes[index++] = dimension.isUseInvertedIndex();
    }
    carbonFactDataHandlerModel.setIsUseInvertedIndex(isUseInvertedIndexes);
    carbonFactDataHandlerModel.setPrimitiveDimLens(segmentProperties.getDimColumnsCardinality());
    carbonFactDataHandlerModel.setBlockSizeInMB(carbonTable.getBlockSizeInMB());
    return carbonFactDataHandlerModel;
}

Example 83 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class StoreCreator method writeDictionary.

private static void writeDictionary(String factFilePath, CarbonTable table) throws Exception {
    BufferedReader reader = new BufferedReader(new FileReader(factFilePath));
    String header = reader.readLine();
    String[] split = header.split(",");
    List<CarbonColumn> allCols = new ArrayList<CarbonColumn>();
    List<CarbonDimension> dims = table.getDimensionByTableName(table.getFactTableName());
    allCols.addAll(dims);
    List<CarbonMeasure> msrs = table.getMeasureByTableName(table.getFactTableName());
    allCols.addAll(msrs);
    Set<String>[] set = new HashSet[dims.size()];
    for (int i = 0; i < set.length; i++) {
        set[i] = new HashSet<String>();
    }
    String line = reader.readLine();
    while (line != null) {
        String[] data = line.split(",");
        for (int i = 0; i < set.length; i++) {
            set[i].add(data[i]);
        }
        line = reader.readLine();
    }
    Cache dictCache = CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY, absoluteTableIdentifier.getStorePath());
    for (int i = 0; i < set.length; i++) {
        ColumnIdentifier columnIdentifier = new ColumnIdentifier(dims.get(i).getColumnId(), null, null);
        CarbonDictionaryWriter writer = new CarbonDictionaryWriterImpl(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier(), columnIdentifier);
        for (String value : set[i]) {
            writer.write(value);
        }
        writer.close();
        writer.commit();
        Dictionary dict = (Dictionary) dictCache.get(new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier.getCarbonTableIdentifier(), columnIdentifier, dims.get(i).getDataType()));
        CarbonDictionarySortInfoPreparator preparator = new CarbonDictionarySortInfoPreparator();
        List<String> newDistinctValues = new ArrayList<String>();
        CarbonDictionarySortInfo dictionarySortInfo = preparator.getDictionarySortInfo(newDistinctValues, dict, dims.get(i).getDataType());
        CarbonDictionarySortIndexWriter carbonDictionaryWriter = new CarbonDictionarySortIndexWriterImpl(absoluteTableIdentifier.getCarbonTableIdentifier(), columnIdentifier, absoluteTableIdentifier.getStorePath());
        try {
            carbonDictionaryWriter.writeSortIndex(dictionarySortInfo.getSortIndex());
            carbonDictionaryWriter.writeInvertedSortIndex(dictionarySortInfo.getSortIndexInverted());
        } finally {
            carbonDictionaryWriter.close();
        }
    }
    reader.close();
}

Example 84 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class CarbonInputFormatUtil method processFilterExpression.

public static void processFilterExpression(Expression filterExpression, CarbonTable carbonTable) {
    List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(carbonTable.getFactTableName());
    List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(carbonTable.getFactTableName());
    QueryModel.processFilterExpression(filterExpression, dimensions, measures);
    if (null != filterExpression) {
        // Optimize Filter Expression and fit RANGE filters is conditions apply.
        FilterOptimizer rangeFilterOptimizer = new RangeFilterOptmizer(new FilterOptimizerBasic(), filterExpression);
        rangeFilterOptimizer.optimizeFilter();
    }
}

Example 85 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class CarbonInputFormatUtil method createQueryPlan.

public static CarbonQueryPlan createQueryPlan(CarbonTable carbonTable, String columnString) {
    String[] columns = null;
    if (columnString != null) {
        columns = columnString.split(",");
    }
    String factTableName = carbonTable.getFactTableName();
    CarbonQueryPlan plan = new CarbonQueryPlan(carbonTable.getDatabaseName(), factTableName);
    // fill dimensions
    // If columns are null, set all dimensions and measures
    int i = 0;
    if (columns != null) {
        for (String column : columns) {
            CarbonDimension dimensionByName = carbonTable.getDimensionByName(factTableName, column);
            if (dimensionByName != null) {
                addQueryDimension(plan, i, dimensionByName);
                i++;
            } else {
                CarbonMeasure measure = carbonTable.getMeasureByName(factTableName, column);
                if (measure == null) {
                    throw new RuntimeException(column + " column not found in the table " + factTableName);
                }
                addQueryMeasure(plan, i, measure);
                i++;
            }
        }
    }
    plan.setQueryId(System.nanoTime() + "");
    return plan;
}