Examples with CarbonMeasure org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure used on opensource projects

Example 71 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class CarbonUtil method getColumnSchemaList.

public static List<ColumnSchema> getColumnSchemaList(List<CarbonDimension> carbonDimensionsList, List<CarbonMeasure> carbonMeasureList) {
    List<ColumnSchema> wrapperColumnSchemaList = new ArrayList<ColumnSchema>();
    fillColumnSchemaListForComplexDims(carbonDimensionsList, wrapperColumnSchemaList);
    for (CarbonMeasure carbonMeasure : carbonMeasureList) {
        wrapperColumnSchemaList.add(carbonMeasure.getColumnSchema());
    }
    return wrapperColumnSchemaList;
}

Example 72 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class BlockletIndexUtil method useMinMaxForBlockletPruning.

/**
 * Method to check whether to serialize min/max values to executor. Returns true if
 * filter column min/max is not cached in driver
 *
 * @param filterResolverTree
 * @param minMaxCacheColumns
 * @return
 */
public static boolean useMinMaxForBlockletPruning(FilterResolverIntf filterResolverTree, List<CarbonColumn> minMaxCacheColumns) {
    boolean serializeMinMax = false;
    if (null != minMaxCacheColumns) {
        Set<CarbonDimension> filterDimensions = new HashSet<>();
        Set<CarbonMeasure> filterMeasures = new HashSet<>();
        QueryUtil.getAllFilterDimensionsAndMeasures(filterResolverTree, filterDimensions, filterMeasures);
        // set flag to true if columns cached size is lesser than filter columns
        if (minMaxCacheColumns.size() < (filterDimensions.size() + filterMeasures.size())) {
            serializeMinMax = true;
        } else {
            // check if all the filter dimensions are cached
            for (CarbonDimension filterDimension : filterDimensions) {
                // cannot validate for complex columns
                if (filterDimension.isComplex()) {
                    continue;
                }
                if (!filterColumnExistsInMinMaxColumnList(minMaxCacheColumns, filterDimension)) {
                    serializeMinMax = true;
                    break;
                }
            }
            // check if all the filter measures are cached only if all filter dimensions are cached
            if (!serializeMinMax) {
                for (CarbonMeasure filterMeasure : filterMeasures) {
                    if (!filterColumnExistsInMinMaxColumnList(minMaxCacheColumns, filterMeasure)) {
                        serializeMinMax = true;
                        break;
                    }
                }
            }
        }
    }
    return serializeMinMax;
}

Example 73 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class CarbonFactDataHandlerModel method getCarbonFactDataHandlerModel.

/**
 * This method will create a model object for carbon fact data handler
 *
 * @param loadModel
 * @return
 */
public static CarbonFactDataHandlerModel getCarbonFactDataHandlerModel(CarbonLoadModel loadModel, CarbonTable carbonTable, SegmentProperties segmentProperties, String tableName, String[] tempStoreLocation, String carbonDataDirectoryPath) {
    // for dynamic page size in write step if varchar columns exist
    List<CarbonDimension> allDimensions = carbonTable.getVisibleDimensions();
    CarbonColumn[] noDicAndComplexColumns = new CarbonColumn[segmentProperties.getNumberOfNoDictionaryDimension() + segmentProperties.getComplexDimensions().size()];
    int noDicAndComp = 0;
    List<DataType> noDictDataTypesList = new ArrayList<>();
    for (CarbonDimension dim : allDimensions) {
        if (dim.getDataType() != DataTypes.DATE) {
            noDicAndComplexColumns[noDicAndComp++] = new CarbonColumn(dim.getColumnSchema(), dim.getOrdinal(), dim.getSchemaOrdinal());
            noDictDataTypesList.add(dim.getDataType());
        }
    }
    CarbonFactDataHandlerModel carbonFactDataHandlerModel = new CarbonFactDataHandlerModel();
    carbonFactDataHandlerModel.setSchemaUpdatedTimeStamp(carbonTable.getTableLastUpdatedTime());
    carbonFactDataHandlerModel.setDatabaseName(loadModel.getDatabaseName());
    carbonFactDataHandlerModel.setTableName(tableName);
    carbonFactDataHandlerModel.setStoreLocation(tempStoreLocation);
    carbonFactDataHandlerModel.setSegmentProperties(segmentProperties);
    carbonFactDataHandlerModel.setSegmentId(loadModel.getSegmentId());
    List<ColumnSchema> wrapperColumnSchema = CarbonUtil.getColumnSchemaList(carbonTable.getVisibleDimensions(), carbonTable.getVisibleMeasures());
    carbonFactDataHandlerModel.setWrapperColumnSchema(wrapperColumnSchema);
    carbonFactDataHandlerModel.setComplexIndexMap(convertComplexDimensionToComplexIndexMap(segmentProperties, loadModel.getSerializationNullFormat()));
    DataType[] measureDataTypes = new DataType[segmentProperties.getMeasures().size()];
    int i = 0;
    for (CarbonMeasure msr : segmentProperties.getMeasures()) {
        measureDataTypes[i++] = msr.getDataType();
    }
    carbonFactDataHandlerModel.setMeasureDataType(measureDataTypes);
    carbonFactDataHandlerModel.setNoDictAndComplexColumns(noDicAndComplexColumns);
    carbonFactDataHandlerModel.setNoDictDataTypesList(noDictDataTypesList);
    CarbonUtil.checkAndCreateFolderWithPermission(carbonDataDirectoryPath);
    carbonFactDataHandlerModel.setCarbonDataDirectoryPath(carbonDataDirectoryPath);
    carbonFactDataHandlerModel.setBlockSizeInMB(carbonTable.getBlockSizeInMB());
    carbonFactDataHandlerModel.setColumnCompressor(loadModel.getColumnCompressor());
    carbonFactDataHandlerModel.tableSpec = new TableSpec(carbonTable, false);
    IndexWriterListener listener = new IndexWriterListener();
    listener.registerAllWriter(carbonTable, loadModel.getSegmentId(), CarbonTablePath.getShardName(CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(loadModel.getTaskNo()), carbonFactDataHandlerModel.getBucketId(), carbonFactDataHandlerModel.getTaskExtension(), String.valueOf(loadModel.getFactTimeStamp()), loadModel.getSegmentId()), segmentProperties);
    carbonFactDataHandlerModel.indexWriterlistener = listener;
    carbonFactDataHandlerModel.initNumberOfCores();
    carbonFactDataHandlerModel.setColumnLocalDictGenMap(CarbonUtil.getLocalDictionaryModel(carbonTable));
    carbonFactDataHandlerModel.sortScope = carbonTable.getSortScope();
    carbonFactDataHandlerModel.setMetrics(loadModel.getMetrics());
    return carbonFactDataHandlerModel;
}

Example 74 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class CarbonDataProcessorUtil method getSchemaColumnNames.

/**
 * This method update the column Name
 *
 * @param schema
 */
public static Set<String> getSchemaColumnNames(CarbonDataLoadSchema schema) {
    Set<String> columnNames = new HashSet<String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    List<CarbonDimension> dimensions = schema.getCarbonTable().getVisibleDimensions();
    Map<String, String> properties = schema.getCarbonTable().getTableInfo().getFactTable().getTableProperties();
    String spatialProperty = properties.get(CarbonCommonConstants.SPATIAL_INDEX);
    spatialProperty = spatialProperty != null ? spatialProperty.trim() : null;
    for (CarbonDimension dimension : dimensions) {
        // skip the non-schema column
        if (!dimension.getColName().equalsIgnoreCase(spatialProperty)) {
            columnNames.add(dimension.getColName());
        }
    }
    List<CarbonMeasure> measures = schema.getCarbonTable().getVisibleMeasures();
    for (CarbonMeasure msr : measures) {
        columnNames.add(msr.getColName());
    }
    return columnNames;
}