Examples with SerializableComparator org.apache.carbondata.core.util.comparator.SerializableComparator used on opensource projects

Example 6 with SerializableComparator

use of org.apache.carbondata.core.util.comparator.SerializableComparator in project carbondata by apache.

the class RowLevelRangeLessThanFiterExecuterImpl method getFilteredIndexesForMeasures.

private BitSet getFilteredIndexesForMeasures(ColumnPage columnPage, int numerOfRows) {
    BitSet bitSet = new BitSet(numerOfRows);
    Object[] filterValues = this.msrFilterRangeValues;
    DataType msrType = msrColEvalutorInfoList.get(0).getType();
    SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrType);
    BitSet nullBitSet = columnPage.getNullBits();
    for (int i = 0; i < filterValues.length; i++) {
        if (filterValues[i] == null) {
            for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) {
                bitSet.set(j);
            }
            continue;
        }
        for (int startIndex = 0; startIndex < numerOfRows; startIndex++) {
            if (!nullBitSet.get(startIndex)) {
                Object msrValue = DataTypeUtil.getMeasureObjectBasedOnDataType(columnPage, startIndex, msrType, msrColEvalutorInfoList.get(0).getMeasure());
                if (comparator.compare(msrValue, filterValues[i]) < 0) {
                    // This is a match.
                    bitSet.set(startIndex);
                }
            }
        }
    }
    return bitSet;
}

Example 7 with SerializableComparator

use of org.apache.carbondata.core.util.comparator.SerializableComparator in project carbondata by apache.

the class PartitionFilterUtil method getPartitionMapForRangeFilter.

/**
 * get partition map of range filter on range partition table
 * @param partitionInfo
 * @param partitioner
 * @param filterValue
 * @param isGreaterThan
 * @param isEqualTo
 * @return
 */
public static BitSet getPartitionMapForRangeFilter(PartitionInfo partitionInfo, RangePartitioner partitioner, Object filterValue, boolean isGreaterThan, boolean isEqualTo, DateFormat timestampFormatter, DateFormat dateFormatter) {
    List<String> values = partitionInfo.getRangeInfo();
    DataType partitionColumnDataType = partitionInfo.getColumnSchemaList().get(0).getDataType();
    SerializableComparator comparator = Comparator.getComparator(partitionColumnDataType);
    BitSet partitionMap = PartitionUtil.generateBitSetBySize(partitioner.numPartitions(), false);
    int numPartitions = values.size();
    int result = 0;
    // the partition index of filter value
    int partitionIndex = 0;
    // find the partition of filter value
    for (; partitionIndex < numPartitions; partitionIndex++) {
        Object value = PartitionUtil.getDataBasedOnDataType(values.get(partitionIndex), partitionColumnDataType, timestampFormatter, dateFormatter);
        if (value instanceof String) {
            value = ByteUtil.toBytes((String) value);
        }
        result = comparator.compare(filterValue, value);
        if (result <= 0) {
            break;
        }
    }
    if (partitionIndex == numPartitions) {
        // filter value is in default partition
        if (isGreaterThan) {
            // GreaterThan(>), GreaterThanEqualTo(>=)
            partitionMap.set(0);
        } else {
            // LessThan(<), LessThanEqualTo(<=)
            partitionMap.set(0, partitioner.numPartitions());
        }
    } else {
        // filter value is not in default partition
        if (result == 0) {
            // if result is 0, the filter value is a bound value of range partition.
            if (isGreaterThan) {
                // GreaterThan(>), GreaterThanEqualTo(>=)
                partitionMap.set(partitionIndex + 2, partitioner.numPartitions());
                partitionMap.set(0);
            } else {
                if (isEqualTo) {
                    // LessThanEqualTo(<=)
                    partitionMap.set(1, partitionIndex + 3);
                } else {
                    // LessThan(<)
                    partitionMap.set(1, partitionIndex + 2);
                }
            }
        } else {
            // the filter value is not a bound value of range partition
            if (isGreaterThan) {
                // GreaterThan(>), GreaterThanEqualTo(>=)
                partitionMap.set(partitionIndex + 1, partitioner.numPartitions());
                partitionMap.set(0);
            } else {
                // LessThan(<), LessThanEqualTo(<=)
                partitionMap.set(1, partitionIndex + 2);
            }
        }
    }
    return partitionMap;
}

Example 8 with SerializableComparator

use of org.apache.carbondata.core.util.comparator.SerializableComparator in project carbondata by apache.

the class CarbonUtil method updateMinMaxValues.

/**
 * This method will be used to update the min and max values and this will be used in case of
 * old store where min and max values for measures are written opposite
 * (i.e max values in place of min and min in place of max values)
 *
 * @param dataFileFooter
 * @param maxValues
 * @param minValues
 * @param isMinValueComparison
 * @return
 */
public static byte[][] updateMinMaxValues(DataFileFooter dataFileFooter, byte[][] maxValues, byte[][] minValues, boolean isMinValueComparison) {
    byte[][] updatedMinMaxValues = new byte[maxValues.length][];
    if (isMinValueComparison) {
        System.arraycopy(minValues, 0, updatedMinMaxValues, 0, minValues.length);
    } else {
        System.arraycopy(maxValues, 0, updatedMinMaxValues, 0, maxValues.length);
    }
    for (int i = 0; i < maxValues.length; i++) {
        // update min and max values only for measures
        if (!dataFileFooter.getColumnInTable().get(i).isDimensionColumn()) {
            DataType dataType = dataFileFooter.getColumnInTable().get(i).getDataType();
            SerializableComparator comparator = Comparator.getComparator(dataType);
            int compare;
            if (isMinValueComparison) {
                compare = comparator.compare(DataTypeUtil.getMeasureObjectFromDataType(maxValues[i], dataType), DataTypeUtil.getMeasureObjectFromDataType(minValues[i], dataType));
                if (compare < 0) {
                    updatedMinMaxValues[i] = maxValues[i];
                }
            } else {
                compare = comparator.compare(DataTypeUtil.getMeasureObjectFromDataType(minValues[i], dataType), DataTypeUtil.getMeasureObjectFromDataType(maxValues[i], dataType));
                if (compare > 0) {
                    updatedMinMaxValues[i] = minValues[i];
                }
            }
        }
    }
    return updatedMinMaxValues;
}

Example 9 with SerializableComparator

use of org.apache.carbondata.core.util.comparator.SerializableComparator in project carbondata by apache.

the class IncludeFilterExecuterImpl method getFilteredIndexesForMsrUsingPrvBitSet.

/**
 * Below method will be used to apply filter on measure column based on previous filtered indexes
 * @param measureColumnPage
 * @param prvBitSetGroup
 * @param pageNumber
 * @param numberOfRows
 * @param msrDataType
 * @return filtred indexes bitset
 */
private BitSet getFilteredIndexesForMsrUsingPrvBitSet(ColumnPage measureColumnPage, BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows, DataType msrDataType) {
    BitSet bitSet = new BitSet(numberOfRows);
    Object[] filterValues = msrColumnExecutorInfo.getFilterKeys();
    BitSet nullBitSet = measureColumnPage.getNullBits();
    BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
    SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrDataType);
    for (int i = 0; i < filterValues.length; i++) {
        if (filterValues[i] == null) {
            for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) {
                bitSet.set(j);
            }
            continue;
        }
        for (int index = prvPageBitSet.nextSetBit(0); index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
            if (!nullBitSet.get(index)) {
                // Check if filterValue[i] matches with measure Values.
                Object msrValue = DataTypeUtil.getMeasureObjectBasedOnDataType(measureColumnPage, index, msrDataType, msrColumnEvaluatorInfo.getMeasure());
                if (comparator.compare(msrValue, filterValues[i]) == 0) {
                    // This is a match.
                    bitSet.set(index);
                }
            }
        }
    }
    return bitSet;
}

Example 10 with SerializableComparator

use of org.apache.carbondata.core.util.comparator.SerializableComparator in project carbondata by apache.

the class IncludeFilterExecuterImpl method getFilteredIndexesForMeasures.

private BitSet getFilteredIndexesForMeasures(ColumnPage columnPage, int rowsInPage, DataType msrType) {
    // Here the algorithm is
    // Get the measure values from the chunk. compare sequentially with the
    // the filter values. The one that matches sets it Bitset.
    BitSet bitSet = new BitSet(rowsInPage);
    Object[] filterValues = msrColumnExecutorInfo.getFilterKeys();
    SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrType);
    BitSet nullBitSet = columnPage.getNullBits();
    for (int i = 0; i < filterValues.length; i++) {
        if (filterValues[i] == null) {
            for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) {
                bitSet.set(j);
            }
            continue;
        }
        for (int startIndex = 0; startIndex < rowsInPage; startIndex++) {
            if (!nullBitSet.get(startIndex)) {
                // Check if filterValue[i] matches with measure Values.
                Object msrValue = DataTypeUtil.getMeasureObjectBasedOnDataType(columnPage, startIndex, msrType, msrColumnEvaluatorInfo.getMeasure());
                if (comparator.compare(msrValue, filterValues[i]) == 0) {
                    // This is a match.
                    bitSet.set(startIndex);
                }
            }
        }
    }
    return bitSet;
}