use of org.apache.carbondata.core.scan.expression.LiteralExpression in project carbondata by apache.
the class CarbondataSplitManager method parseFilterExpression.
/**
* Convert presto-TupleDomain predication into Carbon scan express condition
* @param originalConstraint presto-TupleDomain
* @param carbonTable
* @return
*/
public Expression parseFilterExpression(TupleDomain<ColumnHandle> originalConstraint, CarbonTable carbonTable) {
ImmutableList.Builder<Expression> filters = ImmutableList.builder();
Domain domain = null;
for (ColumnHandle c : originalConstraint.getDomains().get().keySet()) {
CarbondataColumnHandle cdch = (CarbondataColumnHandle) c;
Type type = cdch.getColumnType();
List<CarbonColumn> ccols = carbonTable.getCreateOrderColumn(carbonTable.getFactTableName());
Optional<CarbonColumn> target = ccols.stream().filter(a -> a.getColName().equals(cdch.getColumnName())).findFirst();
if (target.get() == null)
return null;
DataType coltype = target.get().getDataType();
ColumnExpression colExpression = new ColumnExpression(cdch.getColumnName(), target.get().getDataType());
//colExpression.setColIndex(cs.getSchemaOrdinal());
colExpression.setDimension(target.get().isDimension());
colExpression.setDimension(carbonTable.getDimensionByName(carbonTable.getFactTableName(), cdch.getColumnName()));
colExpression.setCarbonColumn(target.get());
domain = originalConstraint.getDomains().get().get(c);
checkArgument(domain.getType().isOrderable(), "Domain type must be orderable");
if (domain.getValues().isNone()) {
}
if (domain.getValues().isAll()) {
}
List<Object> singleValues = new ArrayList<>();
List<Expression> rangeFilter = new ArrayList<>();
for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
// Already checked
checkState(!range.isAll());
if (range.isSingleValue()) {
singleValues.add(range.getLow().getValue());
} else {
List<String> rangeConjuncts = new ArrayList<>();
if (!range.getLow().isLowerUnbounded()) {
Object value = ConvertDataByType(range.getLow().getValue(), type);
switch(range.getLow().getBound()) {
case ABOVE:
if (type == TimestampType.TIMESTAMP) {
//todo not now
} else {
GreaterThanExpression greater = new GreaterThanExpression(colExpression, new LiteralExpression(value, coltype));
rangeFilter.add(greater);
}
break;
case EXACTLY:
GreaterThanEqualToExpression greater = new GreaterThanEqualToExpression(colExpression, new LiteralExpression(value, coltype));
rangeFilter.add(greater);
break;
case BELOW:
throw new IllegalArgumentException("Low marker should never use BELOW bound");
default:
throw new AssertionError("Unhandled bound: " + range.getLow().getBound());
}
}
if (!range.getHigh().isUpperUnbounded()) {
Object value = ConvertDataByType(range.getHigh().getValue(), type);
switch(range.getHigh().getBound()) {
case ABOVE:
throw new IllegalArgumentException("High marker should never use ABOVE bound");
case EXACTLY:
LessThanEqualToExpression less = new LessThanEqualToExpression(colExpression, new LiteralExpression(value, coltype));
rangeFilter.add(less);
break;
case BELOW:
LessThanExpression less2 = new LessThanExpression(colExpression, new LiteralExpression(value, coltype));
rangeFilter.add(less2);
break;
default:
throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
}
}
}
}
if (singleValues.size() == 1) {
Expression ex = null;
if (coltype.equals(DataType.STRING)) {
ex = new EqualToExpression(colExpression, new LiteralExpression(((Slice) singleValues.get(0)).toStringUtf8(), coltype));
} else
ex = new EqualToExpression(colExpression, new LiteralExpression(singleValues.get(0), coltype));
filters.add(ex);
} else if (singleValues.size() > 1) {
ListExpression candidates = null;
List<Expression> exs = singleValues.stream().map((a) -> {
return new LiteralExpression(ConvertDataByType(a, type), coltype);
}).collect(Collectors.toList());
candidates = new ListExpression(exs);
if (candidates != null)
filters.add(new InExpression(colExpression, candidates));
} else if (rangeFilter.size() > 0) {
if (rangeFilter.size() > 1) {
Expression finalFilters = new OrExpression(rangeFilter.get(0), rangeFilter.get(1));
if (rangeFilter.size() > 2) {
for (int i = 2; i < rangeFilter.size(); i++) {
filters.add(new AndExpression(finalFilters, rangeFilter.get(i)));
}
}
} else if (//only have one value
rangeFilter.size() == 1)
filters.add(rangeFilter.get(0));
}
}
Expression finalFilters;
List<Expression> tmp = filters.build();
if (tmp.size() > 1) {
finalFilters = new AndExpression(tmp.get(0), tmp.get(1));
if (tmp.size() > 2) {
for (int i = 2; i < tmp.size(); i++) {
finalFilters = new AndExpression(finalFilters, tmp.get(i));
}
}
} else if (tmp.size() == 1)
finalFilters = tmp.get(0);
else
//no filter
return null;
return finalFilters;
}
use of org.apache.carbondata.core.scan.expression.LiteralExpression in project carbondata by apache.
the class ObjectSerializationUtilTest method testConvertStringToObject.
@Test
public void testConvertStringToObject() throws Exception {
Expression expression = new EqualToExpression(new ColumnExpression("c1", DataType.STRING), new LiteralExpression("a", DataType.STRING));
String string = ObjectSerializationUtil.convertObjectToString(expression);
Assert.assertTrue(string != null);
Object object = ObjectSerializationUtil.convertStringToObject(string);
Assert.assertTrue(object != null);
Assert.assertTrue(object instanceof Expression);
}
use of org.apache.carbondata.core.scan.expression.LiteralExpression in project carbondata by apache.
the class RangeFilterProcessorTest method createFilterTree.
@Test
public void createFilterTree() {
Expression inputFilter;
boolean result = false;
ColumnExpression cola = new ColumnExpression("a", DataType.STRING);
cola.setDimension(true);
ColumnSchema empColumnSchema = new ColumnSchema();
empColumnSchema.setColumnName("empNameCol");
empColumnSchema.setColumnUniqueId("empNameCol");
empColumnSchema.setDimensionColumn(true);
empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
empColumnSchema.setDataType(DataType.STRING);
CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
cola.setDimension(empDimension);
Expression greaterThan = new GreaterThanEqualToExpression(cola, new LiteralExpression("11", DataType.STRING));
ColumnExpression colb = new ColumnExpression("a", DataType.STRING);
colb.setDimension(true);
colb.setDimension(empDimension);
Expression lessThan = new LessThanEqualToExpression(colb, new LiteralExpression("20", DataType.STRING));
inputFilter = new AndExpression(greaterThan, lessThan);
Expression output = new AndExpression(new RangeExpression(new GreaterThanEqualToExpression(new ColumnExpression("a", DataType.STRING), new LiteralExpression("11", DataType.STRING)), new LessThanEqualToExpression(new ColumnExpression("a", DataType.STRING), new LiteralExpression("20", DataType.STRING))), new TrueExpression(null));
FilterOptimizer rangeFilterOptimizer = new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
rangeFilterOptimizer.optimizeFilter();
result = checkBothTrees(inputFilter, output);
Assert.assertTrue(result);
}
use of org.apache.carbondata.core.scan.expression.LiteralExpression in project carbondata by apache.
the class RangeFilterProcessorTest method createFilterTree_flavor1.
@Test
public void createFilterTree_flavor1() {
// Build 3rd BTree a >= '11' and a > '12' and a <= '20' and a <= '15'
Expression inputFilter;
boolean result = false;
ColumnSchema empColumnSchema = new ColumnSchema();
empColumnSchema.setColumnName("a");
empColumnSchema.setColumnUniqueId("a");
empColumnSchema.setDimensionColumn(true);
empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
empColumnSchema.setDataType(DataType.STRING);
CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
ColumnExpression cola1 = new ColumnExpression("a", DataType.STRING);
cola1.setDimension(true);
cola1.setDimension(empDimension);
ColumnExpression cola2 = new ColumnExpression("a", DataType.STRING);
cola2.setDimension(true);
cola2.setDimension(empDimension);
ColumnExpression cola3 = new ColumnExpression("a", DataType.STRING);
cola3.setDimension(true);
cola3.setDimension(empDimension);
ColumnExpression cola4 = new ColumnExpression("a", DataType.STRING);
cola4.setDimension(true);
cola4.setDimension(empDimension);
Expression lessThan1 = new LessThanEqualToExpression(cola1, new LiteralExpression("15", DataType.STRING));
Expression lessThan2 = new LessThanEqualToExpression(cola2, new LiteralExpression("20", DataType.STRING));
Expression greaterThan1 = new GreaterThanExpression(cola3, new LiteralExpression("12", DataType.STRING));
Expression greaterThan2 = new GreaterThanEqualToExpression(cola4, new LiteralExpression("11", DataType.STRING));
Expression And1 = new AndExpression(new NotEqualsExpression(null, null), greaterThan2);
Expression And2 = new AndExpression(And1, greaterThan1);
Expression And3 = new AndExpression(And2, lessThan2);
inputFilter = new AndExpression(And3, lessThan1);
// Build The output
ColumnExpression colb1 = new ColumnExpression("a", DataType.STRING);
cola1.setDimension(true);
cola1.setDimension(empDimension);
ColumnExpression colb2 = new ColumnExpression("a", DataType.STRING);
cola2.setDimension(true);
cola2.setDimension(empDimension);
Expression greaterThanb1 = new GreaterThanExpression(cola3, new LiteralExpression("12", DataType.STRING));
Expression lessThanb1 = new LessThanEqualToExpression(cola1, new LiteralExpression("15", DataType.STRING));
Expression Andb1 = new AndExpression(new NotEqualsExpression(null, null), new TrueExpression(null));
Expression Andb2 = new AndExpression(Andb1, new RangeExpression(greaterThanb1, lessThanb1));
Expression Andb3 = new AndExpression(Andb2, new TrueExpression(null));
FilterOptimizer rangeFilterOptimizer = new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
rangeFilterOptimizer.optimizeFilter();
result = checkBothTrees(inputFilter, new AndExpression(Andb3, new TrueExpression(null)));
// no change
Assert.assertTrue(result);
}
use of org.apache.carbondata.core.scan.expression.LiteralExpression in project carbondata by apache.
the class FilterUtilTest method testIsExpressionNeedsToResolved.
@Test
public void testIsExpressionNeedsToResolved() {
boolean isIncludeFilter = true;
Object obj = "test";
LiteralExpression literalExpression = new LiteralExpression(obj, DataType.STRING);
boolean result = FilterUtil.isExpressionNeedsToResolved(literalExpression, isIncludeFilter);
assertFalse(result);
}
Aggregations