Examples with DirectDictionaryGenerator org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator used on opensource projects

Example 1 with DirectDictionaryGenerator

use of org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator in project carbondata by apache.

the class PrimitiveQueryType method getDataBasedOnDataTypeFromSurrogates.

@Override
public Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData) {
    byte[] data = new byte[keySize];
    surrogateData.get(data);
    Bits bit = new Bits(new int[] { keySize * 8 });
    int surrgateValue = (int) bit.getKeyArray(data, 0)[0];
    Object actualData = null;
    if (isDirectDictionary) {
        DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(dataType);
        actualData = directDictionaryGenerator.getValueFromSurrogate(surrgateValue);
    } else {
        String dictionaryValueForKey = dictionary.getDictionaryValueForKey(surrgateValue);
        actualData = DataTypeUtil.getDataBasedOnDataType(dictionaryValueForKey, this.dataType);
    }
    return actualData;
}

Example 2 with DirectDictionaryGenerator

use of org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator in project carbondata by apache.

the class RowLevelRangeFilterResolverImpl method getDirectSurrogateValues.

private List<Integer> getDirectSurrogateValues(ColumnExpression columnExpression) throws FilterUnsupportedException {
    List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20);
    DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(columnExpression.getDimension().getDataType());
    if (this.getFilterExpression() instanceof BinaryConditionalExpression) {
        listOfExpressionResults = ((BinaryConditionalExpression) this.getFilterExpression()).getLiterals();
    }
    List<Integer> filterValuesList = new ArrayList<Integer>(20);
    try {
        // system can display inconsistent result.
        for (ExpressionResult result : listOfExpressionResults) {
            filterValuesList.add(directDictionaryGenerator.generateDirectSurrogateKey(result.getString(), CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
        }
    } catch (FilterIllegalMemberException e) {
        throw new FilterUnsupportedException(e);
    }
    return filterValuesList;
}

Example 3 with DirectDictionaryGenerator

use of org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator in project carbondata by apache.

the class RestructureUtil method getDirectDictionaryDefaultValue.

/**
 * Method for computing default value for direct dictionary
 *
 * @param dataType
 * @param defaultValue
 * @return
 */
private static Object getDirectDictionaryDefaultValue(DataType dataType, byte[] defaultValue) {
    Object directDictionaryDefaultValue = null;
    if (!isDefaultValueNull(defaultValue)) {
        DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(dataType);
        if (directDictionaryGenerator != null) {
            String value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
            directDictionaryDefaultValue = directDictionaryGenerator.getValueFromSurrogate(Integer.parseInt(value));
        }
    }
    return directDictionaryDefaultValue;
}

Example 4 with DirectDictionaryGenerator

use of org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator in project carbondata by apache.

the class PrimitiveQueryTypeTest method getDataBasedOnDataTypeFromSurrogates.

public Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData) {
    int keySize = 2;
    byte[] data = new byte[keySize];
    surrogateData.get(data);
    Bits bit = new Bits(new int[] { keySize * 8 });
    int surrgateValue = (int) bit.getKeyArray(data, 0)[0];
    Object actualData = null;
    if (isDirectDictionary) {
        DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataTypes.TIMESTAMP);
        actualData = directDictionaryGenerator.getValueFromSurrogate(surrgateValue);
    } else {
        String dictionaryValueForKey = dictionary.getDictionaryValueForKey(surrgateValue);
        actualData = DataTypeUtil.getDataBasedOnDataType(dictionaryValueForKey, DataTypes.TIMESTAMP);
    }
    return actualData;
}

Example 5 with DirectDictionaryGenerator

use of org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator in project carbondata by apache.

the class VectorizedCarbonRecordReader method initBatch.

/**
 * Returns the ColumnarBatch object that will be used for all rows returned by this reader.
 * This object is reused. Calling this enables the vectorized reader. This should be called
 * before any calls to nextKeyValue/nextBatch.
 */
private void initBatch(MemoryMode memMode) {
    List<ProjectionDimension> queryDimension = queryModel.getProjectionDimensions();
    List<ProjectionMeasure> queryMeasures = queryModel.getProjectionMeasures();
    StructField[] fields = new StructField[queryDimension.size() + queryMeasures.size()];
    for (int i = 0; i < queryDimension.size(); i++) {
        ProjectionDimension dim = queryDimension.get(i);
        if (dim.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
            DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(dim.getDimension().getDataType());
            fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(generator.getReturnType()), true, null);
        } else if (!dim.getDimension().hasEncoding(Encoding.DICTIONARY)) {
            fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true, null);
        } else if (dim.getDimension().isComplex()) {
            fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true, null);
        } else {
            fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(DataTypes.INT), true, null);
        }
    }
    for (int i = 0; i < queryMeasures.size(); i++) {
        ProjectionMeasure msr = queryMeasures.get(i);
        DataType dataType = msr.getMeasure().getDataType();
        if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.SHORT || dataType == DataTypes.INT || dataType == DataTypes.LONG) {
            fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(msr.getMeasure().getDataType()), true, null);
        } else if (DataTypes.isDecimal(dataType)) {
            fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), new DecimalType(msr.getMeasure().getPrecision(), msr.getMeasure().getScale()), true, null);
        } else {
            fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(DataTypes.DOUBLE), true, null);
        }
    }
    columnarBatch = ColumnarBatch.allocate(new StructType(fields), memMode);
    CarbonColumnVector[] vectors = new CarbonColumnVector[fields.length];
    boolean[] filteredRows = new boolean[columnarBatch.capacity()];
    for (int i = 0; i < fields.length; i++) {
        vectors[i] = new ColumnarVectorWrapper(columnarBatch.column(i), filteredRows);
    }
    carbonColumnarBatch = new CarbonColumnarBatch(vectors, columnarBatch.capacity(), filteredRows);
}