Examples with CarbonMeasure org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure used on opensource projects

Example 56 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class RowLevelFilterExecutorImpl method initMeasureChunkIndexes.

/**
 * This method will initialize the measure info for the current block to be
 * used for filtering the data
 */
private void initMeasureChunkIndexes() {
    for (int i = 0; i < msrColEvalutorInfoList.size(); i++) {
        // find the measure in the current block measures list
        CarbonMeasure measureFromCurrentBlock = segmentProperties.getMeasureFromCurrentBlock(msrColEvalutorInfoList.get(i).getMeasure());
        if (null != measureFromCurrentBlock) {
            msrColEvalutorInfoList.get(i).setColumnIndex(measureFromCurrentBlock.getOrdinal());
            this.measureChunkIndex[i] = msrColEvalutorInfoList.get(i).getColumnIndexInMinMaxByteArray();
            isMeasurePresentInCurrentBlock[i] = true;
        }
    }
}

Example 57 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class RestructureEvaluatorImpl method isMeasureDefaultValuePresentInFilterValues.

/**
 * This method will check whether a default value for the non-existing column is present
 * in the filter values list
 *
 * @param measureColumnResolvedFilterInfo
 * @return
 */
protected boolean isMeasureDefaultValuePresentInFilterValues(MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo) {
    boolean isDefaultValuePresentInFilterValues = false;
    ColumnFilterInfo filterValues = measureColumnResolvedFilterInfo.getFilterValues();
    CarbonMeasure measure = measureColumnResolvedFilterInfo.getMeasure();
    SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType());
    Object defaultValue = null;
    if (null != measure.getDefaultValue()) {
        // default value for case where user gives is Null condition
        defaultValue = RestructureUtil.getMeasureDefaultValue(measure.getColumnSchema(), measure.getDefaultValue());
    }
    List<Object> measureFilterValuesList = filterValues.getMeasuresFilterValuesList();
    for (Object filterValue : measureFilterValuesList) {
        int compare = comparator.compare(defaultValue, filterValue);
        if (compare == 0) {
            isDefaultValuePresentInFilterValues = true;
            break;
        }
    }
    return isDefaultValuePresentInFilterValues;
}

Example 58 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class RowLevelRangeGreaterThanEqualFilterExecutorImpl method ifDefaultValueMatchesFilter.

/**
 * This method will check whether default value is present in the given filter values
 */
private void ifDefaultValueMatchesFilter() {
    if (!dimColEvaluatorInfoList.isEmpty() && !isDimensionPresentInCurrentBlock[0]) {
        CarbonDimension dimension = this.dimColEvaluatorInfoList.get(0).getDimension();
        byte[] defaultValue = dimension.getDefaultValue();
        if (null != defaultValue) {
            for (int k = 0; k < filterRangeValues.length; k++) {
                int maxCompare = FilterUtil.compareValues(filterRangeValues[k], defaultValue, dimension, false);
                if (maxCompare <= 0) {
                    isDefaultValuePresentInFilter = true;
                    break;
                }
            }
        }
    } else if (!msrColEvalutorInfoList.isEmpty() && !isMeasurePresentInCurrentBlock[0]) {
        CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure();
        byte[] defaultValue = measure.getDefaultValue();
        SerializableComparator comparatorTmp = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType());
        if (null != defaultValue) {
            for (int k = 0; k < msrFilterRangeValues.length; k++) {
                int maxCompare = comparatorTmp.compare(msrFilterRangeValues[k], RestructureUtil.getMeasureDefaultValue(measure.getColumnSchema(), measure.getDefaultValue()));
                if (maxCompare <= 0) {
                    isDefaultValuePresentInFilter = true;
                    break;
                }
            }
        }
    }
}

Example 59 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class StreamRecordReader method initialize.

@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException {
    // input
    if (split instanceof CarbonInputSplit) {
        fileSplit = (CarbonInputSplit) split;
    } else if (split instanceof CarbonMultiBlockSplit) {
        fileSplit = ((CarbonMultiBlockSplit) split).getAllSplits().get(0);
    } else {
        fileSplit = (FileSplit) split;
    }
    // metadata
    hadoopConf = context.getConfiguration();
    if (model == null) {
        CarbonTableInputFormat<Object> format = new CarbonTableInputFormat<>();
        model = format.createQueryModel(split, context);
    }
    carbonTable = model.getTable();
    List<CarbonDimension> dimensions = carbonTable.getVisibleDimensions();
    dimensionCount = dimensions.size();
    List<CarbonMeasure> measures = carbonTable.getVisibleMeasures();
    measureCount = measures.size();
    List<CarbonColumn> carbonColumnList = carbonTable.getStreamStorageOrderColumn();
    storageColumns = carbonColumnList.toArray(new CarbonColumn[carbonColumnList.size()]);
    isNoDictColumn = CarbonDataProcessorUtil.getNoDictionaryMapping(storageColumns);
    directDictionaryGenerators = new DirectDictionaryGenerator[storageColumns.length];
    for (int i = 0; i < storageColumns.length; i++) {
        if (storageColumns[i].getDataType() == DataTypes.DATE) {
            directDictionaryGenerators[i] = DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(storageColumns[i].getDataType());
        }
    }
    dimensionsIsVarcharTypeMap = new boolean[dimensionCount];
    for (int i = 0; i < dimensionCount; i++) {
        dimensionsIsVarcharTypeMap[i] = storageColumns[i].getDataType() == DataTypes.VARCHAR;
    }
    measureDataTypes = new DataType[measureCount];
    for (int i = 0; i < measureCount; i++) {
        measureDataTypes[i] = storageColumns[dimensionCount + i].getDataType();
    }
    // decode data
    allNonNull = new BitSet(storageColumns.length);
    projection = model.getProjectionColumns();
    isRequired = new boolean[storageColumns.length];
    boolean[] isFilterDimensions = model.getIsFilterDimensions();
    boolean[] isFilterMeasures = model.getIsFilterMeasures();
    isFilterRequired = new boolean[storageColumns.length];
    filterMap = new int[storageColumns.length];
    for (int i = 0; i < storageColumns.length; i++) {
        if (storageColumns[i].isDimension()) {
            if (isFilterDimensions[storageColumns[i].getOrdinal()]) {
                isRequired[i] = true;
                isFilterRequired[i] = true;
                filterMap[i] = storageColumns[i].getOrdinal();
            }
        } else {
            if (isFilterMeasures[storageColumns[i].getOrdinal()]) {
                isRequired[i] = true;
                isFilterRequired[i] = true;
                filterMap[i] = carbonTable.getDimensionOrdinalMax() + storageColumns[i].getOrdinal();
            }
        }
    }
    isProjectionRequired = new boolean[storageColumns.length];
    projectionMap = new int[storageColumns.length];
    for (int j = 0; j < projection.length; j++) {
        for (int i = 0; i < storageColumns.length; i++) {
            if (storageColumns[i].getColName().equals(projection[j].getColName())) {
                isRequired[i] = true;
                isProjectionRequired[i] = true;
                projectionMap[i] = j;
                break;
            }
        }
    }
    // initialize filter
    if (null != model.getIndexFilter()) {
        initializeFilter();
    } else if (projection.length == 0) {
        skipScanData = true;
    }
}

Example 60 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class CarbonUtilTest method testToGetColumnSchemaList.

@Test
public void testToGetColumnSchemaList() {
    ColumnSchema column1Schema = new ColumnSchema();
    ColumnSchema column2Schema = new ColumnSchema();
    column1Schema.setColumnName("Column1");
    column2Schema.setColumnName("Column2");
    List<CarbonDimension> carbonDimension = new ArrayList<>();
    carbonDimension.add(new CarbonDimension(column1Schema, 1, 1, 1));
    carbonDimension.add(new CarbonDimension(column2Schema, 2, 2, 1));
    List<CarbonMeasure> carbonMeasure = new ArrayList<>();
    carbonMeasure.add(new CarbonMeasure(column1Schema, 1));
    carbonMeasure.add(new CarbonMeasure(column2Schema, 2));
    List<ColumnSchema> columnSchema = CarbonUtil.getColumnSchemaList(carbonDimension, carbonMeasure);
    for (int i = 0; i < carbonMeasure.size(); i++) {
        assertEquals(columnSchema.get(i), carbonMeasure.get(i).getColumnSchema());
    }
}