Examples with DataType org.apache.carbondata.core.metadata.datatype.DataType used on opensource projects

Example 16 with DataType

use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.

the class CarbondataSplitManager method parseFilterExpression.

/**
   * Convert presto-TupleDomain predication into Carbon scan express condition
   * @param originalConstraint  presto-TupleDomain
   * @param carbonTable
   * @return
   */
public Expression parseFilterExpression(TupleDomain<ColumnHandle> originalConstraint, CarbonTable carbonTable) {
    ImmutableList.Builder<Expression> filters = ImmutableList.builder();
    Domain domain = null;
    for (ColumnHandle c : originalConstraint.getDomains().get().keySet()) {
        CarbondataColumnHandle cdch = (CarbondataColumnHandle) c;
        Type type = cdch.getColumnType();
        List<CarbonColumn> ccols = carbonTable.getCreateOrderColumn(carbonTable.getFactTableName());
        Optional<CarbonColumn> target = ccols.stream().filter(a -> a.getColName().equals(cdch.getColumnName())).findFirst();
        if (target.get() == null)
            return null;
        DataType coltype = target.get().getDataType();
        ColumnExpression colExpression = new ColumnExpression(cdch.getColumnName(), target.get().getDataType());
        //colExpression.setColIndex(cs.getSchemaOrdinal());
        colExpression.setDimension(target.get().isDimension());
        colExpression.setDimension(carbonTable.getDimensionByName(carbonTable.getFactTableName(), cdch.getColumnName()));
        colExpression.setCarbonColumn(target.get());
        domain = originalConstraint.getDomains().get().get(c);
        checkArgument(domain.getType().isOrderable(), "Domain type must be orderable");
        if (domain.getValues().isNone()) {
        }
        if (domain.getValues().isAll()) {
        }
        List<Object> singleValues = new ArrayList<>();
        List<Expression> rangeFilter = new ArrayList<>();
        for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
            // Already checked
            checkState(!range.isAll());
            if (range.isSingleValue()) {
                singleValues.add(range.getLow().getValue());
            } else {
                List<String> rangeConjuncts = new ArrayList<>();
                if (!range.getLow().isLowerUnbounded()) {
                    Object value = ConvertDataByType(range.getLow().getValue(), type);
                    switch(range.getLow().getBound()) {
                        case ABOVE:
                            if (type == TimestampType.TIMESTAMP) {
                            //todo not now
                            } else {
                                GreaterThanExpression greater = new GreaterThanExpression(colExpression, new LiteralExpression(value, coltype));
                                rangeFilter.add(greater);
                            }
                            break;
                        case EXACTLY:
                            GreaterThanEqualToExpression greater = new GreaterThanEqualToExpression(colExpression, new LiteralExpression(value, coltype));
                            rangeFilter.add(greater);
                            break;
                        case BELOW:
                            throw new IllegalArgumentException("Low marker should never use BELOW bound");
                        default:
                            throw new AssertionError("Unhandled bound: " + range.getLow().getBound());
                    }
                }
                if (!range.getHigh().isUpperUnbounded()) {
                    Object value = ConvertDataByType(range.getHigh().getValue(), type);
                    switch(range.getHigh().getBound()) {
                        case ABOVE:
                            throw new IllegalArgumentException("High marker should never use ABOVE bound");
                        case EXACTLY:
                            LessThanEqualToExpression less = new LessThanEqualToExpression(colExpression, new LiteralExpression(value, coltype));
                            rangeFilter.add(less);
                            break;
                        case BELOW:
                            LessThanExpression less2 = new LessThanExpression(colExpression, new LiteralExpression(value, coltype));
                            rangeFilter.add(less2);
                            break;
                        default:
                            throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
                    }
                }
            }
        }
        if (singleValues.size() == 1) {
            Expression ex = null;
            if (coltype.equals(DataType.STRING)) {
                ex = new EqualToExpression(colExpression, new LiteralExpression(((Slice) singleValues.get(0)).toStringUtf8(), coltype));
            } else
                ex = new EqualToExpression(colExpression, new LiteralExpression(singleValues.get(0), coltype));
            filters.add(ex);
        } else if (singleValues.size() > 1) {
            ListExpression candidates = null;
            List<Expression> exs = singleValues.stream().map((a) -> {
                return new LiteralExpression(ConvertDataByType(a, type), coltype);
            }).collect(Collectors.toList());
            candidates = new ListExpression(exs);
            if (candidates != null)
                filters.add(new InExpression(colExpression, candidates));
        } else if (rangeFilter.size() > 0) {
            if (rangeFilter.size() > 1) {
                Expression finalFilters = new OrExpression(rangeFilter.get(0), rangeFilter.get(1));
                if (rangeFilter.size() > 2) {
                    for (int i = 2; i < rangeFilter.size(); i++) {
                        filters.add(new AndExpression(finalFilters, rangeFilter.get(i)));
                    }
                }
            } else if (//only have one value
            rangeFilter.size() == 1)
                filters.add(rangeFilter.get(0));
        }
    }
    Expression finalFilters;
    List<Expression> tmp = filters.build();
    if (tmp.size() > 1) {
        finalFilters = new AndExpression(tmp.get(0), tmp.get(1));
        if (tmp.size() > 2) {
            for (int i = 2; i < tmp.size(); i++) {
                finalFilters = new AndExpression(finalFilters, tmp.get(i));
            }
        }
    } else if (tmp.size() == 1)
        finalFilters = tmp.get(0);
    else
        //no filter
        return null;
    return finalFilters;
}

Example 17 with DataType

use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.

the class ValueCompressionUtil method getDoubleCompressorFinder.

private static CompressionFinder getDoubleCompressorFinder(Object maxValue, Object minValue, int mantissa, byte dataTypeSelected, DataType measureStoreType) {
    //Here we should use the Max abs as max to getDatatype, let's say -1 and -10000000, -1 is max,
    //but we can't use -1 to getDatatype, we should use -10000000.
    double absMaxValue = Math.abs((double) maxValue) >= Math.abs((double) minValue) ? (double) maxValue : (double) minValue;
    DataType adaptiveDataType = getDataType(absMaxValue, mantissa, dataTypeSelected);
    DataType deltaDataType = getDataType((double) maxValue - (double) minValue, mantissa, dataTypeSelected);
    if (mantissa == 0) {
        // short, int, long
        int adaptiveSize = getSize(adaptiveDataType);
        int deltaSize = getSize(deltaDataType);
        if (adaptiveSize > deltaSize) {
            return new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, DataType.DOUBLE, deltaDataType, measureStoreType);
        } else if (adaptiveSize < deltaSize) {
            return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DOUBLE, deltaDataType, measureStoreType);
        } else {
            return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DOUBLE, adaptiveDataType, measureStoreType);
        }
    } else {
        // double
        DataType maxNonDecDataType = getDataType(Math.pow(10, mantissa) * absMaxValue, 0, dataTypeSelected);
        DataType diffNonDecDataType = getDataType(Math.pow(10, mantissa) * ((double) maxValue - (double) minValue), 0, dataTypeSelected);
        CompressionFinder[] finders = { new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, adaptiveDataType, adaptiveDataType, CompressionFinder.PRIORITY.ACTUAL, measureStoreType), new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, adaptiveDataType, deltaDataType, CompressionFinder.PRIORITY.DIFFSIZE, measureStoreType), new CompressionFinder(COMPRESSION_TYPE.BIGINT, adaptiveDataType, maxNonDecDataType, CompressionFinder.PRIORITY.MAXNONDECIMAL, measureStoreType), new CompressionFinder(COMPRESSION_TYPE.DELTA_NON_DECIMAL, adaptiveDataType, diffNonDecDataType, CompressionFinder.PRIORITY.DIFFNONDECIMAL, measureStoreType) };
        // sort the compressionFinder.The top have the highest priority
        Arrays.sort(finders);
        return finders[0];
    }
}

Example 18 with DataType

use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.

the class RangeExpressionEvaluator method addFilterExpressionMap.

private void addFilterExpressionMap(Map<String, List<FilterModificationNode>> filterExpressionMap, Expression currentNode, Expression parentNode) {
    String colName = getColumnName(currentNode);
    DataType dataType = getLiteralDataType(currentNode);
    Object literalVal = getLiteralValue(currentNode);
    ExpressionType expType = getExpressionType(currentNode);
    FilterModificationNode filterExpression = new FilterModificationNode(currentNode, parentNode, expType, dataType, literalVal, colName);
    if (null == filterExpressionMap.get(colName)) {
        filterExpressionMap.put(colName, new ArrayList<FilterModificationNode>());
    }
    filterExpressionMap.get(colName).add(filterExpression);
}

Example 19 with DataType

use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.

the class ValueCompressionUtilTest method testToGetValueCompressionModelForByteAndIntAndDataTypeSelected0.

@Test
public void testToGetValueCompressionModelForByteAndIntAndDataTypeSelected0() {
    Object[] maxValues = { -32766.00 };
    Object[] minValues = { 32744.0 };
    int[] decimalLength = { 1 };
    Object[] uniqueValues = { 5 };
    DataType[] types = { DataType.DOUBLE };
    byte[] dataTypeSelected = { 0 };
    MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types, dataTypeSelected);
    WriterCompressModel writerCompressModel = ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
    assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, writerCompressModel.getCompType(0));
}

Example 20 with DataType

use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.

the class ValueCompressionUtilTest method testToGetValueCompressionModelForDefaultAggregatorType.

@Test
public void testToGetValueCompressionModelForDefaultAggregatorType() {
    Object[] maxValues = { 10.0 };
    Object[] minValues = { 1.0 };
    int[] decimalLength = { 0 };
    Object[] uniqueValues = { 5 };
    DataType[] types = { DataType.DOUBLE };
    byte[] dataTypeSelected = { 1 };
    MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types, dataTypeSelected);
    WriterCompressModel writerCompressModel = ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
    assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, writerCompressModel.getCompType(0));
}