Examples with DecimalType org.apache.carbondata.core.metadata.datatype.DecimalType used on opensource projects

Example 6 with DecimalType

use of org.apache.carbondata.core.metadata.datatype.DecimalType in project carbondata by apache.

the class AbstractDataFileFooterConverter method thriftColumnSchmeaToWrapperColumnSchema.

protected ColumnSchema thriftColumnSchmeaToWrapperColumnSchema(org.apache.carbondata.format.ColumnSchema externalColumnSchema) {
    ColumnSchema wrapperColumnSchema = new ColumnSchema();
    wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id());
    wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name());
    wrapperColumnSchema.setColumnar(externalColumnSchema.isColumnar());
    DataType dataType = thriftDataTyopeToWrapperDataType(externalColumnSchema.data_type);
    if (DataTypes.isDecimal(dataType)) {
        DecimalType decimalType = (DecimalType) dataType;
        decimalType.setPrecision(externalColumnSchema.getPrecision());
        decimalType.setScale(externalColumnSchema.getScale());
    }
    wrapperColumnSchema.setDataType(dataType);
    wrapperColumnSchema.setDimensionColumn(externalColumnSchema.isDimension());
    List<Encoding> encoders = new ArrayList<Encoding>();
    for (org.apache.carbondata.format.Encoding encoder : externalColumnSchema.getEncoders()) {
        encoders.add(fromExternalToWrapperEncoding(encoder));
    }
    wrapperColumnSchema.setEncodingList(encoders);
    wrapperColumnSchema.setNumberOfChild(externalColumnSchema.getNum_child());
    wrapperColumnSchema.setPrecision(externalColumnSchema.getPrecision());
    wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id());
    wrapperColumnSchema.setScale(externalColumnSchema.getScale());
    wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());
    Map<String, String> properties = externalColumnSchema.getColumnProperties();
    if (properties != null) {
        if (properties.get(CarbonCommonConstants.SORT_COLUMNS) != null) {
            wrapperColumnSchema.setSortColumn(true);
        }
    }
    wrapperColumnSchema.setFunction(externalColumnSchema.getAggregate_function());
    List<org.apache.carbondata.format.ParentColumnTableRelation> parentColumnTableRelation = externalColumnSchema.getParentColumnTableRelations();
    if (null != parentColumnTableRelation) {
        wrapperColumnSchema.setParentColumnTableRelations(fromThriftToWrapperParentTableColumnRelations(parentColumnTableRelation));
    }
    return wrapperColumnSchema;
}

Example 7 with DecimalType

use of org.apache.carbondata.core.metadata.datatype.DecimalType in project carbondata by apache.

the class ColumnPageEncoderMeta method readFields.

@Override
public void readFields(DataInput in) throws IOException {
    columnSpec = new TableSpec.ColumnSpec();
    columnSpec.readFields(in);
    storeDataType = DataTypes.valueOf(in.readByte());
    if (DataTypes.isDecimal(storeDataType)) {
        DecimalType decimalType = (DecimalType) storeDataType;
        decimalType.setPrecision(columnSpec.getPrecision());
        decimalType.setScale(columnSpec.getScale());
    }
    setDecimal(in.readInt());
    setDataTypeSelected(in.readByte());
    readMinMax(in);
    compressorName = in.readUTF();
}

Example 8 with DecimalType

use of org.apache.carbondata.core.metadata.datatype.DecimalType in project carbondata by apache.

the class CSVCarbonWriterTest method testWriteJsonSchemaWithDefaultDecimal.

@Test
public void testWriteJsonSchemaWithDefaultDecimal() {
    String jsonSchema = new StringBuilder().append("[ \n").append("   {\"name\":\"string\"},\n").append("   {\"age\":\"int\"},\n").append("   {\"height\":\"double\"},\n").append("   {\"decimalField\":\"decimal\"}\n").append("]").toString();
    Schema schema = Schema.parseJson(jsonSchema);
    assert (10 == ((DecimalType) schema.getFields()[3].getDataType()).getPrecision());
    assert (2 == ((DecimalType) schema.getFields()[3].getDataType()).getScale());
}

Example 9 with DecimalType

use of org.apache.carbondata.core.metadata.datatype.DecimalType in project carbondata by apache.

the class TableSchemaBuilder method addColumn.

private ColumnSchema addColumn(StructField field, String parentName, AtomicInteger valIndex, boolean isSortColumn, boolean isComplexChild, boolean isInvertedIdxColumn) {
    Objects.requireNonNull(field);
    if (isComplexChild) {
        // if field is complex then append parent name to the child field to check
        // if any other field with same name exists
        checkRepeatColumnName(field, parentName);
    } else {
        checkRepeatColumnName(field);
    }
    ColumnSchema newColumn = new ColumnSchema();
    if (parentName != null) {
        newColumn.setColumnName(parentName + "." + field.getFieldName());
    } else {
        newColumn.setColumnName(field.getFieldName());
    }
    newColumn.setDataType(field.getDataType());
    newColumn.setDimensionColumn(isSortColumn || field.getDataType() == DataTypes.STRING || field.getDataType() == DataTypes.VARCHAR || field.getDataType() == DataTypes.DATE || field.getDataType() == DataTypes.TIMESTAMP || field.getDataType() == DataTypes.BINARY || field.getDataType().isComplexType() || (isComplexChild));
    if (!isComplexChild) {
        newColumn.setSchemaOrdinal(ordinal++);
    } else {
        // child column should not be counted for schema ordinal
        newColumn.setSchemaOrdinal(-1);
    }
    // For NonTransactionalTable, multiple sdk writer output with same column name can be placed in
    // single folder for query.
    // That time many places in code, columnId check will fail. To avoid that
    // keep column ID as same as column name.
    // Anyhow Alter table is not supported for NonTransactionalTable.
    // SO, this will not have any impact.
    newColumn.setColumnUniqueId(field.getFieldName());
    newColumn.setColumnReferenceId(newColumn.getColumnUniqueId());
    newColumn.setEncodingList(createEncoding(field.getDataType(), isInvertedIdxColumn, isComplexChild));
    if (field.getDataType().isComplexType()) {
        if (DataTypes.isArrayType(field.getDataType()) || DataTypes.isMapType(field.getDataType())) {
            newColumn.setNumberOfChild(1);
        } else {
            newColumn.setNumberOfChild(((StructType) field.getDataType()).getFields().size());
        }
    }
    if (DataTypes.isDecimal(field.getDataType())) {
        DecimalType decimalType = (DecimalType) field.getDataType();
        newColumn.setPrecision(decimalType.getPrecision());
        newColumn.setScale(decimalType.getScale());
    }
    if (!isSortColumn) {
        if (!newColumn.isDimensionColumn()) {
            measures.add(newColumn);
        } else if (DataTypes.isStructType(field.getDataType()) || DataTypes.isArrayType(field.getDataType()) || DataTypes.isMapType(field.getDataType()) || isComplexChild) {
            complex.add(newColumn);
        } else {
            if (field.getDataType() == DataTypes.VARCHAR) {
                varCharColumns.add(newColumn);
            } else {
                dimension.add(newColumn);
            }
        }
    } else {
        newColumn.setSortColumn(true);
        sortColumns.add(newColumn);
    }
    if (field.getDataType().isComplexType()) {
        String parentFieldName = newColumn.getColumnName();
        if (DataTypes.isArrayType(field.getDataType())) {
            for (StructField structField : field.getChildren()) {
                String colName = getColNameForArray(valIndex);
                if (null != ((ArrayType) field.getDataType()).getElementName()) {
                    colName = ((ArrayType) field.getDataType()).getElementName();
                }
                structField.setFieldName(colName);
                addColumn(structField, parentFieldName, valIndex, false, true, isInvertedIdxColumn);
            }
        } else if (DataTypes.isStructType(field.getDataType()) && ((StructType) field.getDataType()).getFields().size() > 0) {
            // This field has children.
            if (field.getChildren() != null) {
                for (StructField structField : field.getChildren()) {
                    addColumn(structField, parentFieldName, valIndex, false, true, isInvertedIdxColumn);
                }
            }
        } else if (DataTypes.isMapType(field.getDataType())) {
            for (StructField structField : field.getChildren()) {
                structField.setFieldName(getColNameForArray(valIndex));
                addColumn(structField, parentFieldName, valIndex, false, true, isInvertedIdxColumn);
            }
        }
    }
    // todo: need more information such as long_string_columns
    return newColumn;
}

Example 10 with DecimalType

use of org.apache.carbondata.core.metadata.datatype.DecimalType in project carbondata by apache.

the class ColumnarVectorWrapperDirectWithDeleteDeltaAndInvertedIndex method convert.

@Override
public void convert() {
    if (columnVector instanceof CarbonColumnVectorImpl) {
        CarbonColumnVectorImpl localVector = (CarbonColumnVectorImpl) columnVector;
        DataType dataType = columnVector.getType();
        int length = invertedIndex.length;
        int counter = 0;
        if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
            byte[] dataArray = (byte[]) localVector.getDataArray();
            for (int i = 0; i < length; i++) {
                if (!deletedRows.get(i)) {
                    if (nullBits.get(i)) {
                        carbonColumnVector.putNull(counter++);
                    } else {
                        carbonColumnVector.putByte(counter++, dataArray[i]);
                    }
                }
            }
        } else if (dataType == DataTypes.SHORT) {
            short[] dataArray = (short[]) localVector.getDataArray();
            for (int i = 0; i < length; i++) {
                if (!deletedRows.get(i)) {
                    if (nullBits.get(i)) {
                        carbonColumnVector.putNull(counter++);
                    } else {
                        carbonColumnVector.putShort(counter++, dataArray[i]);
                    }
                }
            }
        } else if (dataType == DataTypes.INT) {
            int[] dataArray = (int[]) localVector.getDataArray();
            for (int i = 0; i < length; i++) {
                if (!deletedRows.get(i)) {
                    if (nullBits.get(i)) {
                        carbonColumnVector.putNull(counter++);
                    } else {
                        carbonColumnVector.putInt(counter++, dataArray[i]);
                    }
                }
            }
        } else if (dataType == DataTypes.LONG || dataType == DataTypes.TIMESTAMP) {
            long[] dataArray = (long[]) localVector.getDataArray();
            for (int i = 0; i < length; i++) {
                if (!deletedRows.get(i)) {
                    if (nullBits.get(i)) {
                        carbonColumnVector.putNull(counter++);
                    } else {
                        carbonColumnVector.putLong(counter++, dataArray[i]);
                    }
                }
            }
        } else if (dataType == DataTypes.FLOAT) {
            float[] dataArray = (float[]) localVector.getDataArray();
            for (int i = 0; i < length; i++) {
                if (!deletedRows.get(i)) {
                    if (nullBits.get(i)) {
                        carbonColumnVector.putNull(counter++);
                    } else {
                        carbonColumnVector.putFloat(counter++, dataArray[i]);
                    }
                }
            }
        } else if (dataType == DataTypes.DOUBLE) {
            double[] dataArray = (double[]) localVector.getDataArray();
            for (int i = 0; i < length; i++) {
                if (!deletedRows.get(i)) {
                    if (nullBits.get(i)) {
                        carbonColumnVector.putNull(counter++);
                    } else {
                        carbonColumnVector.putDouble(counter++, dataArray[i]);
                    }
                }
            }
        } else if (dataType instanceof DecimalType) {
            BigDecimal[] dataArray = (BigDecimal[]) localVector.getDataArray();
            for (int i = 0; i < length; i++) {
                if (!deletedRows.get(i)) {
                    if (nullBits.get(i)) {
                        carbonColumnVector.putNull(counter++);
                    } else {
                        carbonColumnVector.putDecimal(counter++, dataArray[i], precision);
                    }
                }
            }
        } else if (dataType == DataTypes.STRING || dataType == DataTypes.BYTE_ARRAY) {
            int[] offsets = localVector.getOffsets();
            int[] lengths = localVector.getLengths();
            if (offsets != null && lengths != null) {
                for (int i = 0; i < length; i++) {
                    if (!deletedRows.get(i)) {
                        if (nullBits.get(i)) {
                            carbonColumnVector.putNull(counter++);
                        } else {
                            carbonColumnVector.putArray(counter++, offsets[i], lengths[i]);
                        }
                    }
                }
            } else {
                byte[][] dataArray = (byte[][]) localVector.getDataArray();
                for (int i = 0; i < length; i++) {
                    if (!deletedRows.get(i)) {
                        if (nullBits.get(i)) {
                            carbonColumnVector.putNull(counter++);
                        } else {
                            carbonColumnVector.putByteArray(counter++, dataArray[i]);
                        }
                    }
                }
            }
        }
    }
}