use of org.apache.carbondata.core.scan.expression.MatchExpression in project carbondata by apache.
the class RowLevelFilterExecuterImpl method applyFilter.
@Override
public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
if (exp instanceof MatchExpression) {
return rawBlockletColumnChunks.getBitSetGroup();
}
readColumnChunks(rawBlockletColumnChunks);
// CHECKSTYLE:ON
int[] numberOfRows = null;
int pageNumbers = 0;
if (dimColEvaluatorInfoList.size() > 0) {
if (isDimensionPresentInCurrentBlock[0]) {
pageNumbers = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]].getPagesCount();
numberOfRows = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]].getRowCount();
} else {
// specific for restructure case where default values need to be filled
pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
numberOfRows = new int[pageNumbers];
for (int i = 0; i < pageNumbers; i++) {
numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
}
}
}
if (msrColEvalutorInfoList.size() > 0) {
if (isMeasurePresentInCurrentBlock[0]) {
pageNumbers = rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]].getPagesCount();
numberOfRows = rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]].getRowCount();
} else {
// specific for restructure case where default values need to be filled
pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
numberOfRows = new int[pageNumbers];
for (int i = 0; i < pageNumbers; i++) {
numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
}
}
}
BitSetGroup bitSetGroup = new BitSetGroup(pageNumbers);
for (int i = 0; i < pageNumbers; i++) {
BitSet set = new BitSet(numberOfRows[i]);
RowIntf row = new RowImpl();
BitSet prvBitset = null;
// otherwise use older flow
if (!useBitsetPipeLine || null == rawBlockletColumnChunks.getBitSetGroup() || null == bitSetGroup.getBitSet(i) || rawBlockletColumnChunks.getBitSetGroup().getBitSet(i).isEmpty()) {
for (int index = 0; index < numberOfRows[i]; index++) {
createRow(rawBlockletColumnChunks, row, i, index);
Boolean rslt = false;
try {
rslt = exp.evaluate(row).getBoolean();
}// too much log inforation only once the log will be printed.
catch (FilterIllegalMemberException e) {
FilterUtil.logError(e, false);
}
if (null != rslt && rslt) {
set.set(index);
}
}
} else {
prvBitset = rawBlockletColumnChunks.getBitSetGroup().getBitSet(i);
for (int index = prvBitset.nextSetBit(0); index >= 0; index = prvBitset.nextSetBit(index + 1)) {
createRow(rawBlockletColumnChunks, row, i, index);
Boolean rslt = false;
try {
rslt = exp.evaluate(row).getBoolean();
} catch (FilterIllegalMemberException e) {
FilterUtil.logError(e, false);
}
if (null != rslt && rslt) {
set.set(index);
}
}
}
bitSetGroup.setBitSet(set, i);
}
return bitSetGroup;
}
use of org.apache.carbondata.core.scan.expression.MatchExpression in project carbondata by apache.
the class RowLevelFilterExecutorImpl method applyFilter.
@Override
public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
if (exp instanceof MatchExpression) {
BitSetGroup bitSetGroup = rawBlockletColumnChunks.getBitSetGroup();
if (bitSetGroup == null) {
// It means there are no index created on this table
throw new FilterUnsupportedException(String.format("%s is not supported on table %s", exp.getFilterExpressionType().name(), tableIdentifier.getTableName()));
}
return bitSetGroup;
}
readColumnChunks(rawBlockletColumnChunks);
// CHECKSTYLE:ON
int[] numberOfRows = null;
int pageNumbers = 0;
if (dimColEvaluatorInfoList.size() > 0) {
if (isDimensionPresentInCurrentBlock[0]) {
pageNumbers = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]].getPagesCount();
numberOfRows = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]].getRowCount();
} else {
// specific for restructure case where default values need to be filled
pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
numberOfRows = new int[pageNumbers];
for (int i = 0; i < pageNumbers; i++) {
numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
}
}
}
if (msrColEvalutorInfoList.size() > 0) {
if (isMeasurePresentInCurrentBlock[0]) {
pageNumbers = rawBlockletColumnChunks.getMeasureRawColumnChunks()[msrColEvalutorInfoList.get(0).getColumnIndex()].getPagesCount();
numberOfRows = rawBlockletColumnChunks.getMeasureRawColumnChunks()[msrColEvalutorInfoList.get(0).getColumnIndex()].getRowCount();
} else {
// specific for restructure case where default values need to be filled
pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
numberOfRows = new int[pageNumbers];
for (int i = 0; i < pageNumbers; i++) {
numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
}
}
}
BitSetGroup bitSetGroup = new BitSetGroup(pageNumbers);
if (isDimensionPresentInCurrentBlock.length == 1 && isDimensionPresentInCurrentBlock[0] && dimColEvaluatorInfoList.get(0).getDimension().getDataType().isComplexType() && exp instanceof EqualToExpression) {
LiteralExpression literalExp = (LiteralExpression) (((EqualToExpression) exp).getRight());
// convert filter value to byte[] to compare with byte[] data from columnPage
Object literalExpValue = literalExp.getLiteralExpValue();
DataType literalExpDataType = literalExp.getLiteralExpDataType();
if (literalExpDataType == DataTypes.TIMESTAMP) {
if ((long) literalExpValue == 0) {
literalExpValue = null;
} else {
literalExpValue = (long) literalExpValue / TimeStampGranularityTypeValue.MILLIS_SECONDS.getValue();
}
} else if (literalExpDataType == DataTypes.DATE) {
// change data type to int to get the byte[] filter value as it is direct dictionary
literalExpDataType = DataTypes.INT;
if (literalExpValue == null) {
literalExpValue = CarbonCommonConstants.DIRECT_DICT_VALUE_NULL;
} else {
literalExpValue = (int) literalExpValue + DateDirectDictionaryGenerator.cutOffDate;
}
}
byte[] filterValueInBytes = DataTypeUtil.getBytesDataDataTypeForNoDictionaryColumn(literalExpValue, literalExpDataType);
ArrayQueryType complexType = (ArrayQueryType) complexDimensionInfoMap.get(dimensionChunkIndex[0]);
int totalCount = 0;
// check all the pages
for (int i = 0; i < pageNumbers; i++) {
if (limit != -1 && totalCount >= limit) {
break;
}
BitSet set = new BitSet(numberOfRows[i]);
int[][] numberOfChild = complexType.getNumberOfChild(rawBlockletColumnChunks.getDimensionRawColumnChunks(), null, numberOfRows[i], i);
DimensionColumnPage page = complexType.parseBlockAndReturnChildData(rawBlockletColumnChunks.getDimensionRawColumnChunks(), null, i);
// check every row
for (int index = 0; index < numberOfRows[i]; index++) {
if (limit != -1 && totalCount >= limit) {
break;
}
int dataOffset = numberOfChild[index][1];
// loop the children
for (int j = 0; j < numberOfChild[index][0]; j++) {
byte[] obj = page.getChunkData(dataOffset++);
if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(obj, filterValueInBytes) == 0) {
set.set(index);
totalCount++;
break;
}
}
}
bitSetGroup.setBitSet(set, i);
}
} else {
for (int i = 0; i < pageNumbers; i++) {
BitSet set = new BitSet(numberOfRows[i]);
RowIntf row = new RowImpl();
BitSet prvBitset = null;
// otherwise use older flow
if (!useBitsetPipeLine || null == rawBlockletColumnChunks.getBitSetGroup() || null == bitSetGroup.getBitSet(i) || rawBlockletColumnChunks.getBitSetGroup().getBitSet(i).isEmpty()) {
for (int index = 0; index < numberOfRows[i]; index++) {
createRow(rawBlockletColumnChunks, row, i, index);
Boolean result = false;
try {
result = exp.evaluate(row).getBoolean();
}// too much log inforation only once the log will be printed.
catch (FilterIllegalMemberException e) {
FilterUtil.logError(e, false);
}
if (null != result && result) {
set.set(index);
}
}
} else {
prvBitset = rawBlockletColumnChunks.getBitSetGroup().getBitSet(i);
for (int index = prvBitset.nextSetBit(0); index >= 0; index = prvBitset.nextSetBit(index + 1)) {
createRow(rawBlockletColumnChunks, row, i, index);
Boolean rslt = false;
try {
rslt = exp.evaluate(row).getBoolean();
} catch (FilterIllegalMemberException e) {
FilterUtil.logError(e, false);
}
if (null != rslt && rslt) {
set.set(index);
}
}
}
bitSetGroup.setBitSet(set, i);
}
}
return bitSetGroup;
}
Aggregations