Examples with DataType org.apache.carbondata.core.metadata.datatype.DataType used on opensource projects

Example 6 with DataType

use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.

the class RowLevelFilterExecuterImpl method createRow.

/**
   * Method will read the members of particular dimension block and create
   * a row instance for further processing of the filters
   *
   * @param blockChunkHolder
   * @param row
   * @param index
   * @throws IOException
   */
private void createRow(BlocksChunkHolder blockChunkHolder, RowIntf row, int pageIndex, int index) throws IOException {
    Object[] record = new Object[dimColEvaluatorInfoList.size() + msrColEvalutorInfoList.size()];
    String memberString;
    for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
        DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
        // if filter dimension is not present in the current add its default value
        if (!isDimensionPresentInCurrentBlock[i]) {
            // fill default value here
            record[dimColumnEvaluatorInfo.getRowIndex()] = getDimensionDefaultValue(dimColumnEvaluatorInfo);
            continue;
        }
        if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY && dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
            if (!dimColumnEvaluatorInfo.isDimensionExistsInCurrentSilce()) {
                record[dimColumnEvaluatorInfo.getRowIndex()] = dimColumnEvaluatorInfo.getDimension().getDefaultValue();
            }
            DimensionColumnDataChunk columnDataChunk = blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]].convertToDimColDataChunk(pageIndex);
            if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY) && columnDataChunk instanceof VariableLengthDimensionDataChunk) {
                VariableLengthDimensionDataChunk dimensionColumnDataChunk = (VariableLengthDimensionDataChunk) columnDataChunk;
                byte[] memberBytes = dimensionColumnDataChunk.getChunkData(index);
                if (null != memberBytes) {
                    if (Arrays.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, memberBytes)) {
                        memberBytes = null;
                    }
                    record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(memberBytes, dimColumnEvaluatorInfo.getDimension().getDataType());
                } else {
                    continue;
                }
            } else {
                int dictionaryValue = readSurrogatesFromColumnBlock(blockChunkHolder, index, pageIndex, dimColumnEvaluatorInfo, dimensionBlocksIndex[i]);
                if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY) && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
                    memberString = getFilterActualValueFromDictionaryValue(dimColumnEvaluatorInfo, dictionaryValue);
                    record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil.getDataBasedOnDataType(memberString, dimColumnEvaluatorInfo.getDimension().getDataType());
                } else if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
                    Object member = getFilterActualValueFromDirectDictionaryValue(dimColumnEvaluatorInfo, dictionaryValue);
                    record[dimColumnEvaluatorInfo.getRowIndex()] = member;
                }
            }
        } else {
            try {
                GenericQueryType complexType = complexDimensionInfoMap.get(dimensionBlocksIndex[i]);
                ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
                DataOutputStream dataOutputStream = new DataOutputStream(byteStream);
                complexType.parseBlocksAndReturnComplexColumnByteArray(blockChunkHolder.getDimensionRawDataChunk(), index, pageIndex, dataOutputStream);
                record[dimColumnEvaluatorInfo.getRowIndex()] = complexType.getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
                byteStream.close();
            } catch (IOException e) {
                LOGGER.info(e.getMessage());
            }
        }
    }
    DataType msrType;
    for (int i = 0; i < msrColEvalutorInfoList.size(); i++) {
        MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = msrColEvalutorInfoList.get(i);
        switch(msrColumnEvalutorInfo.getType()) {
            case SHORT:
                msrType = DataType.SHORT;
                break;
            case INT:
                msrType = DataType.INT;
                break;
            case LONG:
                msrType = DataType.LONG;
                break;
            case DECIMAL:
                msrType = DataType.DECIMAL;
                break;
            default:
                msrType = DataType.DOUBLE;
        }
        // in the current block measure list
        if (!isMeasurePresentInCurrentBlock[i]) {
            byte[] defaultValue = msrColumnEvalutorInfo.getCarbonColumn().getDefaultValue();
            record[msrColumnEvalutorInfo.getRowIndex()] = RestructureUtil.getMeasureDefaultValue(msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema(), defaultValue);
            continue;
        }
        Object msrValue;
        MeasureColumnDataChunk measureColumnDataChunk = blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]].convertToMeasureColDataChunk(pageIndex);
        switch(msrType) {
            case SHORT:
                msrValue = (short) measureColumnDataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
                break;
            case INT:
                msrValue = (int) measureColumnDataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
                break;
            case LONG:
                msrValue = measureColumnDataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
                break;
            case DECIMAL:
                BigDecimal bigDecimalValue = measureColumnDataChunk.getMeasureDataHolder().getReadableBigDecimalValueByIndex(index);
                if (null != bigDecimalValue && msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale() > bigDecimalValue.scale()) {
                    bigDecimalValue = bigDecimalValue.setScale(msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale(), RoundingMode.HALF_UP);
                }
                msrValue = bigDecimalValue;
                break;
            default:
                msrValue = measureColumnDataChunk.getMeasureDataHolder().getReadableDoubleValueByIndex(index);
        }
        record[msrColumnEvalutorInfo.getRowIndex()] = measureColumnDataChunk.getNullValueIndexHolder().getBitSet().get(index) ? null : msrValue;
    }
    row.setValues(record);
}

Example 7 with DataType

use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.

the class PartitionFilterUtil method getPartitionMapForRangeFilter.

/**
   * get partition map of range filter on list partition table
   * @param partitionInfo
   * @param partitioner
   * @param filterValue
   * @param isGreaterThan
   * @param isEqualTo
   * @return
   */
public static BitSet getPartitionMapForRangeFilter(PartitionInfo partitionInfo, ListPartitioner partitioner, Object filterValue, boolean isGreaterThan, boolean isEqualTo) {
    List<List<String>> values = partitionInfo.getListInfo();
    DataType partitionColumnDataType = partitionInfo.getColumnSchemaList().get(0).getDataType();
    Comparator comparator = PartitionFilterUtil.getComparatorByDataType(partitionColumnDataType);
    BitSet partitionMap = PartitionUtil.generateBitSetBySize(partitioner.numPartitions(), false);
    // add default partition
    partitionMap.set(partitioner.numPartitions() - 1);
    int partitions = values.size();
    if (isGreaterThan) {
        if (isEqualTo) {
            // GreaterThanEqualTo(>=)
            outer1: for (int i = 0; i < partitions; i++) {
                for (String value : values.get(i)) {
                    Object listValue = PartitionUtil.getDataBasedOnDataType(value, partitionColumnDataType);
                    if (comparator.compare(listValue, filterValue) >= 0) {
                        partitionMap.set(i);
                        continue outer1;
                    }
                }
            }
        } else {
            // GreaterThan(>)
            outer2: for (int i = 0; i < partitions; i++) {
                for (String value : values.get(i)) {
                    Object listValue = PartitionUtil.getDataBasedOnDataType(value, partitionColumnDataType);
                    if (comparator.compare(listValue, filterValue) > 0) {
                        partitionMap.set(i);
                        continue outer2;
                    }
                }
            }
        }
    } else {
        if (isEqualTo) {
            // LessThanEqualTo(<=)
            outer3: for (int i = 0; i < partitions; i++) {
                for (String value : values.get(i)) {
                    Object listValue = PartitionUtil.getDataBasedOnDataType(value, partitionColumnDataType);
                    if (comparator.compare(listValue, filterValue) <= 0) {
                        partitionMap.set(i);
                        continue outer3;
                    }
                }
            }
        } else {
            // LessThanEqualTo(<)
            outer4: for (int i = 0; i < partitions; i++) {
                for (String value : values.get(i)) {
                    Object listValue = PartitionUtil.getDataBasedOnDataType(value, partitionColumnDataType);
                    if (comparator.compare(listValue, filterValue) < 0) {
                        partitionMap.set(i);
                        continue outer4;
                    }
                }
            }
        }
    }
    return partitionMap;
}

Example 8 with DataType

use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.

the class PartitionFilterUtil method getPartitionMapForRangeFilter.

/**
   * get partition map of range filter on range partition table
   * @param partitionInfo
   * @param partitioner
   * @param filterValue
   * @param isGreaterThan
   * @param isEqualTo
   * @return
   */
public static BitSet getPartitionMapForRangeFilter(PartitionInfo partitionInfo, RangePartitioner partitioner, Object filterValue, boolean isGreaterThan, boolean isEqualTo) {
    List<String> values = partitionInfo.getRangeInfo();
    DataType partitionColumnDataType = partitionInfo.getColumnSchemaList().get(0).getDataType();
    Comparator comparator = PartitionFilterUtil.getComparatorByDataType(partitionColumnDataType);
    BitSet partitionMap = PartitionUtil.generateBitSetBySize(partitioner.numPartitions(), false);
    int numPartitions = values.size();
    int result = 0;
    // the partition index of filter value
    int partitionIndex = 0;
    // find the partition of filter value
    for (; partitionIndex < numPartitions; partitionIndex++) {
        result = comparator.compare(filterValue, PartitionUtil.getDataBasedOnDataType(values.get(partitionIndex), partitionColumnDataType));
        if (result <= 0) {
            break;
        }
    }
    if (partitionIndex == numPartitions) {
        // filter value is in default partition
        if (isGreaterThan) {
            // GreaterThan(>), GreaterThanEqualTo(>=)
            partitionMap.set(numPartitions);
        } else {
            // LessThan(<), LessThanEqualTo(<=)
            partitionMap.set(0, partitioner.numPartitions());
        }
    } else {
        // filter value is not in default partition
        if (result == 0) {
            // if result is 0, the filter value is a bound value of range partition.
            if (isGreaterThan) {
                // GreaterThan(>), GreaterThanEqualTo(>=)
                partitionMap.set(partitionIndex + 1, partitioner.numPartitions());
            } else {
                if (isEqualTo) {
                    // LessThanEqualTo(<=)
                    partitionMap.set(0, partitionIndex + 2);
                } else {
                    // LessThan(<)
                    partitionMap.set(0, partitionIndex + 1);
                }
            }
        } else {
            // the filter value is not a bound value of range partition
            if (isGreaterThan) {
                // GreaterThan(>), GreaterThanEqualTo(>=)
                partitionMap.set(partitionIndex, partitioner.numPartitions());
            } else {
                // LessThan(<), LessThanEqualTo(<=)
                partitionMap.set(0, partitionIndex + 1);
            }
        }
    }
    return partitionMap;
}

Example 9 with DataType

use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.

the class ColumnExpressionTest method setUp.

@Before
public void setUp() {
    String columnName = "IMEI";
    DataType dataType = DataType.STRING;
    columnExpression = new ColumnExpression(columnName, dataType);
}

Example 10 with DataType

use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.

the class ValueCompressionUtilTest method testToGetValueCompressionModelForByteAndInt.

@Test
public void testToGetValueCompressionModelForByteAndInt() {
    Object[] maxValues = { -32766.00 };
    Object[] minValues = { 32744.0 };
    int[] decimalLength = { 0 };
    Object[] uniqueValues = { 5 };
    DataType[] types = { DataType.DOUBLE };
    byte[] dataTypeSelected = { 1 };
    MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types, dataTypeSelected);
    WriterCompressModel writerCompressModel = ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
    assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, writerCompressModel.getCompType(0));
}