Examples with DataType org.apache.carbondata.core.metadata.datatype.DataType used on opensource projects

Example 11 with DataType

use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.

the class ValueCompressionUtilTest method testToGetValueCompressionModel.

@Test
public void testToGetValueCompressionModel() {
    Object[] maxValues = { 10L, 20L, 30L };
    Object[] minValues = { 1L, 2L, 3L };
    int[] decimalLength = { 0, 0, 0 };
    Object[] uniqueValues = { 5, new Long[] { 2L, 4L }, 2L };
    DataType[] types = { DataType.LONG, DataType.LONG, DataType.LONG };
    byte[] dataTypeSelected = { 1, 2, 4 };
    MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 3, uniqueValues, types, dataTypeSelected);
    WriterCompressModel writerCompressModel = ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
    assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, writerCompressModel.getCompType(0));
    assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, writerCompressModel.getCompType(1));
    assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, writerCompressModel.getCompType(2));
}

Example 12 with DataType

use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.

the class ValueCompressionUtilTest method testToGetValueCompressionModelForIntAndShort.

@Test
public void testToGetValueCompressionModelForIntAndShort() {
    Object[] maxValues = { 1111111111.0 };
    Object[] minValues = { 1111078433.0 };
    int[] decimalLength = { 0 };
    Object[] uniqueValues = { 5 };
    DataType[] types = { DataType.DOUBLE };
    byte[] dataTypeSelected = { 1 };
    MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types, dataTypeSelected);
    WriterCompressModel writerCompressModel = ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
    assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, writerCompressModel.getCompType(0));
}

Example 13 with DataType

use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.

the class SortParameters method createSortParameters.

public static SortParameters createSortParameters(CarbonTable carbonTable, String databaseName, String tableName, int dimColCount, int complexDimColCount, int measureColCount, int noDictionaryCount, String partitionID, String segmentId, String taskNo, boolean[] noDictionaryColMaping, boolean isCompactionFlow) {
    SortParameters parameters = new SortParameters();
    CarbonProperties carbonProperties = CarbonProperties.getInstance();
    parameters.setDatabaseName(databaseName);
    parameters.setTableName(tableName);
    parameters.setPartitionID(partitionID);
    parameters.setSegmentId(segmentId);
    parameters.setTaskNo(taskNo);
    parameters.setMeasureColCount(measureColCount);
    parameters.setDimColCount(dimColCount - complexDimColCount);
    parameters.setNumberOfSortColumns(carbonTable.getNumberOfSortColumns());
    parameters.setNoDictionaryCount(noDictionaryCount);
    parameters.setNumberOfNoDictSortColumns(carbonTable.getNumberOfNoDictSortColumns());
    parameters.setComplexDimColCount(complexDimColCount);
    parameters.setNoDictionaryDimnesionColumn(noDictionaryColMaping);
    parameters.setObserver(new SortObserver());
    // get sort buffer size
    parameters.setSortBufferSize(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.SORT_SIZE, CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL)));
    LOGGER.info("Sort size for table: " + parameters.getSortBufferSize());
    // set number of intermedaite file to merge
    parameters.setNumberOfIntermediateFileToBeMerged(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT, CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE)));
    LOGGER.info("Number of intermediate file to be merged: " + parameters.getNumberOfIntermediateFileToBeMerged());
    // get file buffer size
    parameters.setFileBufferSize(CarbonDataProcessorUtil.getFileBufferSize(parameters.getNumberOfIntermediateFileToBeMerged(), carbonProperties, CarbonCommonConstants.CONSTANT_SIZE_TEN));
    LOGGER.info("File Buffer Size: " + parameters.getFileBufferSize());
    String carbonDataDirectoryPath = CarbonDataProcessorUtil.getLocalDataFolderLocation(databaseName, tableName, taskNo, partitionID, segmentId, isCompactionFlow);
    parameters.setTempFileLocation(carbonDataDirectoryPath + File.separator + CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
    LOGGER.info("temp file location" + parameters.getTempFileLocation());
    int numberOfCores;
    try {
        numberOfCores = Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.NUM_CORES_LOADING, CarbonCommonConstants.NUM_CORES_DEFAULT_VAL));
        numberOfCores = numberOfCores / 2;
    } catch (NumberFormatException exc) {
        numberOfCores = Integer.parseInt(CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
    }
    parameters.setNumberOfCores(numberOfCores > 0 ? numberOfCores : 1);
    parameters.setFileWriteBufferSize(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE, CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE_DEFAULT_VALUE)));
    parameters.setSortFileCompressionEnabled(Boolean.parseBoolean(carbonProperties.getProperty(CarbonCommonConstants.IS_SORT_TEMP_FILE_COMPRESSION_ENABLED, CarbonCommonConstants.IS_SORT_TEMP_FILE_COMPRESSION_ENABLED_DEFAULTVALUE)));
    int sortTempFileNoOFRecordsInCompression;
    try {
        sortTempFileNoOFRecordsInCompression = Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION, CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE));
        if (sortTempFileNoOFRecordsInCompression < 1) {
            LOGGER.error("Invalid value for: " + CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION + ":Only Positive Integer value(greater than zero) is allowed.Default value will " + "be used");
            sortTempFileNoOFRecordsInCompression = Integer.parseInt(CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE);
        }
    } catch (NumberFormatException e) {
        LOGGER.error("Invalid value for: " + CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION + ", only Positive Integer value is allowed. Default value will be used");
        sortTempFileNoOFRecordsInCompression = Integer.parseInt(CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE);
    }
    parameters.setSortTempFileNoOFRecordsInCompression(sortTempFileNoOFRecordsInCompression);
    if (parameters.isSortFileCompressionEnabled()) {
        LOGGER.info("Compression will be used for writing the sort temp File");
    }
    parameters.setPrefetch(CarbonCommonConstants.CARBON_PREFETCH_IN_MERGE_VALUE);
    parameters.setBufferSize(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE, CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE_DEFAULT)));
    DataType[] type = CarbonDataProcessorUtil.getMeasureDataType(parameters.getMeasureColCount(), parameters.getDatabaseName(), parameters.getTableName());
    parameters.setMeasureDataType(type);
    setNoDictionarySortColumnMapping(parameters);
    return parameters;
}

Example 14 with DataType

use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.

the class CarbonDataProcessorUtil method getMeasureDataType.

public static DataType[] getMeasureDataType(int measureCount, String databaseName, String tableName) {
    DataType[] type = new DataType[measureCount];
    for (int i = 0; i < type.length; i++) {
        type[i] = DataType.DOUBLE;
    }
    CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName + CarbonCommonConstants.UNDERSCORE + tableName);
    List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(tableName);
    for (int i = 0; i < type.length; i++) {
        type[i] = measures.get(i).getDataType();
    }
    return type;
}

Example 15 with DataType

use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.

the class CarbondataRecordSetProvider method fillFilter2QueryModel.

// Build filter for QueryModel
private void fillFilter2QueryModel(QueryModel queryModel, TupleDomain<ColumnHandle> originalConstraint, CarbonTable carbonTable) {
    //queryModel.setFilterExpressionResolverTree(new FilterResolverIntf());
    //Build Predicate Expression
    ImmutableList.Builder<Expression> filters = ImmutableList.builder();
    Domain domain = null;
    for (ColumnHandle c : originalConstraint.getDomains().get().keySet()) {
        // Build ColumnExpresstion for Expresstion(Carbondata)
        CarbondataColumnHandle cdch = (CarbondataColumnHandle) c;
        Type type = cdch.getColumnType();
        DataType coltype = Spi2CarbondataTypeMapper(cdch);
        Expression colExpression = new ColumnExpression(cdch.getColumnName(), coltype);
        domain = originalConstraint.getDomains().get().get(c);
        checkArgument(domain.getType().isOrderable(), "Domain type must be orderable");
        if (domain.getValues().isNone()) {
        }
        if (domain.getValues().isAll()) {
        }
        List<Object> singleValues = new ArrayList<>();
        List<Expression> rangeFilter = new ArrayList<>();
        for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
            // Already checked
            checkState(!range.isAll());
            if (range.isSingleValue()) {
                singleValues.add(range.getLow().getValue());
            } else {
                List<String> rangeConjuncts = new ArrayList<>();
                if (!range.getLow().isLowerUnbounded()) {
                    Object value = ConvertDataByType(range.getLow().getValue(), type);
                    switch(range.getLow().getBound()) {
                        case ABOVE:
                            if (type == TimestampType.TIMESTAMP) {
                            //todo not now
                            } else {
                                GreaterThanExpression greater = new GreaterThanExpression(colExpression, new LiteralExpression(value, coltype));
                                rangeFilter.add(greater);
                            }
                            break;
                        case EXACTLY:
                            GreaterThanEqualToExpression greater = new GreaterThanEqualToExpression(colExpression, new LiteralExpression(value, coltype));
                            rangeFilter.add(greater);
                            break;
                        case BELOW:
                            throw new IllegalArgumentException("Low marker should never use BELOW bound");
                        default:
                            throw new AssertionError("Unhandled bound: " + range.getLow().getBound());
                    }
                }
                if (!range.getHigh().isUpperUnbounded()) {
                    Object value = ConvertDataByType(range.getHigh().getValue(), type);
                    switch(range.getHigh().getBound()) {
                        case ABOVE:
                            throw new IllegalArgumentException("High marker should never use ABOVE bound");
                        case EXACTLY:
                            LessThanEqualToExpression less = new LessThanEqualToExpression(colExpression, new LiteralExpression(value, coltype));
                            rangeFilter.add(less);
                            break;
                        case BELOW:
                            LessThanExpression less2 = new LessThanExpression(colExpression, new LiteralExpression(value, coltype));
                            rangeFilter.add(less2);
                            break;
                        default:
                            throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
                    }
                }
            }
        }
        if (singleValues.size() == 1) {
            Expression ex = null;
            if (coltype.equals(DataType.STRING)) {
                ex = new EqualToExpression(colExpression, new LiteralExpression(((Slice) singleValues.get(0)).toStringUtf8(), coltype));
            } else if (coltype.equals(DataType.TIMESTAMP) || coltype.equals(DataType.DATE)) {
                Long value = (Long) singleValues.get(0) * 1000;
                ex = new EqualToExpression(colExpression, new LiteralExpression(value, coltype));
            } else
                ex = new EqualToExpression(colExpression, new LiteralExpression(singleValues.get(0), coltype));
            filters.add(ex);
        } else if (singleValues.size() > 1) {
            ListExpression candidates = null;
            List<Expression> exs = singleValues.stream().map((a) -> {
                return new LiteralExpression(ConvertDataByType(a, type), coltype);
            }).collect(Collectors.toList());
            candidates = new ListExpression(exs);
            if (candidates != null)
                filters.add(new InExpression(colExpression, candidates));
        } else if (rangeFilter.size() > 0) {
            if (rangeFilter.size() > 1) {
                Expression finalFilters = new OrExpression(rangeFilter.get(0), rangeFilter.get(1));
                if (rangeFilter.size() > 2) {
                    for (int i = 2; i < rangeFilter.size(); i++) {
                        filters.add(new AndExpression(finalFilters, rangeFilter.get(i)));
                    }
                }
            } else if (rangeFilter.size() == 1)
                filters.add(rangeFilter.get(0));
        }
    }
    Expression finalFilters;
    List<Expression> tmp = filters.build();
    if (tmp.size() > 1) {
        finalFilters = new AndExpression(tmp.get(0), tmp.get(1));
        if (tmp.size() > 2) {
            for (int i = 2; i < tmp.size(); i++) {
                finalFilters = new AndExpression(finalFilters, tmp.get(i));
            }
        }
    } else if (tmp.size() == 1)
        finalFilters = tmp.get(0);
    else
        return;
    // todo set into QueryModel
    CarbonInputFormatUtil.processFilterExpression(finalFilters, carbonTable);
    queryModel.setFilterExpressionResolverTree(CarbonInputFormatUtil.resolveFilter(finalFilters, queryModel.getAbsoluteTableIdentifier()));
}