use of org.apache.hadoop.hive.common.type.HiveDecimal in project hive by apache.
the class VectorizationContext method getInExpression.
/**
* Create a filter or boolean-valued expression for column IN ( <list-of-constants> )
*/
private VectorExpression getInExpression(List<ExprNodeDesc> childExpr, VectorExpressionDescriptor.Mode mode, TypeInfo returnType) throws HiveException {
ExprNodeDesc colExpr = childExpr.get(0);
List<ExprNodeDesc> inChildren = childExpr.subList(1, childExpr.size());
String colType = colExpr.getTypeString();
colType = VectorizationContext.mapTypeNameSynonyms(colType);
TypeInfo colTypeInfo = TypeInfoUtils.getTypeInfoFromTypeString(colType);
Category category = colTypeInfo.getCategory();
if (category == Category.STRUCT) {
return getStructInExpression(childExpr, colExpr, colTypeInfo, inChildren, mode, returnType);
} else if (category != Category.PRIMITIVE) {
return null;
}
// prepare arguments for createVectorExpression
List<ExprNodeDesc> childrenForInList = evaluateCastOnConstants(inChildren);
/* This method assumes that the IN list has no NULL entries. That is enforced elsewhere,
* in the Vectorizer class. If NULL is passed in as a list entry, behavior is not defined.
* If in the future, NULL values are allowed in the IN list, be sure to handle 3-valued
* logic correctly. E.g. NOT (col IN (null)) should be considered UNKNOWN, so that would
* become FALSE in the WHERE clause, and cause the row in question to be filtered out.
* See the discussion in Jira HIVE-5583.
*/
VectorExpression expr = null;
// Validate the IN items are only constants.
for (ExprNodeDesc inListChild : childrenForInList) {
if (!(inListChild instanceof ExprNodeConstantDesc)) {
throw new HiveException("Vectorizing IN expression only supported for constant values");
}
}
// determine class
Class<?> cl = null;
// non-vectorized validates that explicitly during UDF init.
if (isIntFamily(colType)) {
cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterLongColumnInList.class : LongColumnInList.class);
long[] inVals = new long[childrenForInList.size()];
for (int i = 0; i != inVals.length; i++) {
inVals[i] = getIntFamilyScalarAsLong((ExprNodeConstantDesc) childrenForInList.get(i));
}
expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType);
((ILongInExpr) expr).setInListValues(inVals);
} else if (isTimestampFamily(colType)) {
cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterTimestampColumnInList.class : TimestampColumnInList.class);
Timestamp[] inVals = new Timestamp[childrenForInList.size()];
for (int i = 0; i != inVals.length; i++) {
inVals[i] = getTimestampScalar(childrenForInList.get(i));
}
expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType);
((ITimestampInExpr) expr).setInListValues(inVals);
} else if (isStringFamily(colType)) {
cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterStringColumnInList.class : StringColumnInList.class);
byte[][] inVals = new byte[childrenForInList.size()][];
for (int i = 0; i != inVals.length; i++) {
inVals[i] = getStringScalarAsByteArray((ExprNodeConstantDesc) childrenForInList.get(i));
}
expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType);
((IStringInExpr) expr).setInListValues(inVals);
} else if (isFloatFamily(colType)) {
cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterDoubleColumnInList.class : DoubleColumnInList.class);
double[] inValsD = new double[childrenForInList.size()];
for (int i = 0; i != inValsD.length; i++) {
inValsD[i] = getNumericScalarAsDouble(childrenForInList.get(i));
}
expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType);
((IDoubleInExpr) expr).setInListValues(inValsD);
} else if (isDecimalFamily(colType)) {
cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterDecimalColumnInList.class : DecimalColumnInList.class);
HiveDecimal[] inValsD = new HiveDecimal[childrenForInList.size()];
for (int i = 0; i != inValsD.length; i++) {
inValsD[i] = (HiveDecimal) getVectorTypeScalarValue((ExprNodeConstantDesc) childrenForInList.get(i));
}
expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType);
((IDecimalInExpr) expr).setInListValues(inValsD);
} else if (isDateFamily(colType)) {
cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterLongColumnInList.class : LongColumnInList.class);
long[] inVals = new long[childrenForInList.size()];
for (int i = 0; i != inVals.length; i++) {
inVals[i] = (Long) getVectorTypeScalarValue((ExprNodeConstantDesc) childrenForInList.get(i));
}
expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType);
((ILongInExpr) expr).setInListValues(inVals);
}
// execution to fall back to row mode.
return expr;
}
use of org.apache.hadoop.hive.common.type.HiveDecimal in project hive by apache.
the class VectorizedRowBatchCtx method addPartitionColsToBatch.
/**
* Add the partition values to the batch
*
* @param batch
* @param partitionValues
* @throws HiveException
*/
public void addPartitionColsToBatch(VectorizedRowBatch batch, Object[] partitionValues) {
if (partitionValues != null) {
for (int i = 0; i < partitionColumnCount; i++) {
Object value = partitionValues[i];
int colIndex = dataColumnCount + i;
String partitionColumnName = rowColumnNames[colIndex];
PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) rowColumnTypeInfos[colIndex];
switch(primitiveTypeInfo.getPrimitiveCategory()) {
case BOOLEAN:
{
LongColumnVector lcv = (LongColumnVector) batch.cols[colIndex];
if (value == null) {
lcv.noNulls = false;
lcv.isNull[0] = true;
lcv.isRepeating = true;
} else {
lcv.fill((Boolean) value == true ? 1 : 0);
lcv.isNull[0] = false;
}
}
break;
case BYTE:
{
LongColumnVector lcv = (LongColumnVector) batch.cols[colIndex];
if (value == null) {
lcv.noNulls = false;
lcv.isNull[0] = true;
lcv.isRepeating = true;
} else {
lcv.fill((Byte) value);
lcv.isNull[0] = false;
}
}
break;
case SHORT:
{
LongColumnVector lcv = (LongColumnVector) batch.cols[colIndex];
if (value == null) {
lcv.noNulls = false;
lcv.isNull[0] = true;
lcv.isRepeating = true;
} else {
lcv.fill((Short) value);
lcv.isNull[0] = false;
}
}
break;
case INT:
{
LongColumnVector lcv = (LongColumnVector) batch.cols[colIndex];
if (value == null) {
lcv.noNulls = false;
lcv.isNull[0] = true;
lcv.isRepeating = true;
} else {
lcv.fill((Integer) value);
lcv.isNull[0] = false;
}
}
break;
case LONG:
{
LongColumnVector lcv = (LongColumnVector) batch.cols[colIndex];
if (value == null) {
lcv.noNulls = false;
lcv.isNull[0] = true;
lcv.isRepeating = true;
} else {
lcv.fill((Long) value);
lcv.isNull[0] = false;
}
}
break;
case DATE:
{
LongColumnVector lcv = (LongColumnVector) batch.cols[colIndex];
if (value == null) {
lcv.noNulls = false;
lcv.isNull[0] = true;
lcv.isRepeating = true;
} else {
lcv.fill(DateWritable.dateToDays((Date) value));
lcv.isNull[0] = false;
}
}
break;
case TIMESTAMP:
{
TimestampColumnVector lcv = (TimestampColumnVector) batch.cols[colIndex];
if (value == null) {
lcv.noNulls = false;
lcv.isNull[0] = true;
lcv.isRepeating = true;
} else {
lcv.fill((Timestamp) value);
lcv.isNull[0] = false;
}
}
break;
case INTERVAL_YEAR_MONTH:
{
LongColumnVector lcv = (LongColumnVector) batch.cols[colIndex];
if (value == null) {
lcv.noNulls = false;
lcv.isNull[0] = true;
lcv.isRepeating = true;
} else {
lcv.fill(((HiveIntervalYearMonth) value).getTotalMonths());
lcv.isNull[0] = false;
}
}
case INTERVAL_DAY_TIME:
{
IntervalDayTimeColumnVector icv = (IntervalDayTimeColumnVector) batch.cols[colIndex];
if (value == null) {
icv.noNulls = false;
icv.isNull[0] = true;
icv.isRepeating = true;
} else {
icv.fill(((HiveIntervalDayTime) value));
icv.isNull[0] = false;
}
}
case FLOAT:
{
DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[colIndex];
if (value == null) {
dcv.noNulls = false;
dcv.isNull[0] = true;
dcv.isRepeating = true;
} else {
dcv.fill((Float) value);
dcv.isNull[0] = false;
}
}
break;
case DOUBLE:
{
DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[colIndex];
if (value == null) {
dcv.noNulls = false;
dcv.isNull[0] = true;
dcv.isRepeating = true;
} else {
dcv.fill((Double) value);
dcv.isNull[0] = false;
}
}
break;
case DECIMAL:
{
DecimalColumnVector dv = (DecimalColumnVector) batch.cols[colIndex];
if (value == null) {
dv.noNulls = false;
dv.isNull[0] = true;
dv.isRepeating = true;
} else {
HiveDecimal hd = (HiveDecimal) value;
dv.set(0, hd);
dv.isRepeating = true;
dv.isNull[0] = false;
}
}
break;
case BINARY:
{
BytesColumnVector bcv = (BytesColumnVector) batch.cols[colIndex];
byte[] bytes = (byte[]) value;
if (bytes == null) {
bcv.noNulls = false;
bcv.isNull[0] = true;
bcv.isRepeating = true;
} else {
bcv.fill(bytes);
bcv.isNull[0] = false;
}
}
break;
case STRING:
case CHAR:
case VARCHAR:
{
BytesColumnVector bcv = (BytesColumnVector) batch.cols[colIndex];
String sVal = value.toString();
if (sVal == null) {
bcv.noNulls = false;
bcv.isNull[0] = true;
bcv.isRepeating = true;
} else {
bcv.setVal(0, sVal.getBytes());
bcv.isRepeating = true;
}
}
break;
default:
throw new RuntimeException("Unable to recognize the partition type " + primitiveTypeInfo.getPrimitiveCategory() + " for column " + partitionColumnName);
}
}
}
}
use of org.apache.hadoop.hive.common.type.HiveDecimal in project hive by apache.
the class VectorUDFAdaptor method setOutputCol.
private void setOutputCol(ColumnVector colVec, int i, Object value) {
/* Depending on the output type, get the value, cast the result to the
* correct type if needed, and assign the result into the output vector.
*/
if (outputOI instanceof WritableStringObjectInspector) {
BytesColumnVector bv = (BytesColumnVector) colVec;
Text t;
if (value instanceof String) {
t = new Text((String) value);
} else {
t = ((WritableStringObjectInspector) outputOI).getPrimitiveWritableObject(value);
}
bv.setVal(i, t.getBytes(), 0, t.getLength());
} else if (outputOI instanceof WritableHiveCharObjectInspector) {
WritableHiveCharObjectInspector writableHiveCharObjectOI = (WritableHiveCharObjectInspector) outputOI;
int maxLength = ((CharTypeInfo) writableHiveCharObjectOI.getTypeInfo()).getLength();
BytesColumnVector bv = (BytesColumnVector) colVec;
HiveCharWritable hiveCharWritable;
if (value instanceof HiveCharWritable) {
hiveCharWritable = ((HiveCharWritable) value);
} else {
hiveCharWritable = writableHiveCharObjectOI.getPrimitiveWritableObject(value);
}
Text t = hiveCharWritable.getTextValue();
// In vector mode, we stored CHAR as unpadded.
StringExpr.rightTrimAndTruncate(bv, i, t.getBytes(), 0, t.getLength(), maxLength);
} else if (outputOI instanceof WritableHiveVarcharObjectInspector) {
WritableHiveVarcharObjectInspector writableHiveVarcharObjectOI = (WritableHiveVarcharObjectInspector) outputOI;
int maxLength = ((VarcharTypeInfo) writableHiveVarcharObjectOI.getTypeInfo()).getLength();
BytesColumnVector bv = (BytesColumnVector) colVec;
HiveVarcharWritable hiveVarcharWritable;
if (value instanceof HiveVarcharWritable) {
hiveVarcharWritable = ((HiveVarcharWritable) value);
} else {
hiveVarcharWritable = writableHiveVarcharObjectOI.getPrimitiveWritableObject(value);
}
Text t = hiveVarcharWritable.getTextValue();
StringExpr.truncate(bv, i, t.getBytes(), 0, t.getLength(), maxLength);
} else if (outputOI instanceof WritableIntObjectInspector) {
LongColumnVector lv = (LongColumnVector) colVec;
if (value instanceof Integer) {
lv.vector[i] = (Integer) value;
} else {
lv.vector[i] = ((WritableIntObjectInspector) outputOI).get(value);
}
} else if (outputOI instanceof WritableLongObjectInspector) {
LongColumnVector lv = (LongColumnVector) colVec;
if (value instanceof Long) {
lv.vector[i] = (Long) value;
} else {
lv.vector[i] = ((WritableLongObjectInspector) outputOI).get(value);
}
} else if (outputOI instanceof WritableDoubleObjectInspector) {
DoubleColumnVector dv = (DoubleColumnVector) colVec;
if (value instanceof Double) {
dv.vector[i] = (Double) value;
} else {
dv.vector[i] = ((WritableDoubleObjectInspector) outputOI).get(value);
}
} else if (outputOI instanceof WritableFloatObjectInspector) {
DoubleColumnVector dv = (DoubleColumnVector) colVec;
if (value instanceof Float) {
dv.vector[i] = (Float) value;
} else {
dv.vector[i] = ((WritableFloatObjectInspector) outputOI).get(value);
}
} else if (outputOI instanceof WritableShortObjectInspector) {
LongColumnVector lv = (LongColumnVector) colVec;
if (value instanceof Short) {
lv.vector[i] = (Short) value;
} else {
lv.vector[i] = ((WritableShortObjectInspector) outputOI).get(value);
}
} else if (outputOI instanceof WritableByteObjectInspector) {
LongColumnVector lv = (LongColumnVector) colVec;
if (value instanceof Byte) {
lv.vector[i] = (Byte) value;
} else {
lv.vector[i] = ((WritableByteObjectInspector) outputOI).get(value);
}
} else if (outputOI instanceof WritableTimestampObjectInspector) {
TimestampColumnVector tv = (TimestampColumnVector) colVec;
Timestamp ts;
if (value instanceof Timestamp) {
ts = (Timestamp) value;
} else {
ts = ((WritableTimestampObjectInspector) outputOI).getPrimitiveJavaObject(value);
}
tv.set(i, ts);
} else if (outputOI instanceof WritableDateObjectInspector) {
LongColumnVector lv = (LongColumnVector) colVec;
Date ts;
if (value instanceof Date) {
ts = (Date) value;
} else {
ts = ((WritableDateObjectInspector) outputOI).getPrimitiveJavaObject(value);
}
long l = DateWritable.dateToDays(ts);
lv.vector[i] = l;
} else if (outputOI instanceof WritableBooleanObjectInspector) {
LongColumnVector lv = (LongColumnVector) colVec;
if (value instanceof Boolean) {
lv.vector[i] = (Boolean) value ? 1 : 0;
} else {
lv.vector[i] = ((WritableBooleanObjectInspector) outputOI).get(value) ? 1 : 0;
}
} else if (outputOI instanceof WritableHiveDecimalObjectInspector) {
DecimalColumnVector dcv = (DecimalColumnVector) colVec;
if (value instanceof HiveDecimal) {
dcv.set(i, (HiveDecimal) value);
} else {
HiveDecimal hd = ((WritableHiveDecimalObjectInspector) outputOI).getPrimitiveJavaObject(value);
dcv.set(i, hd);
}
} else if (outputOI instanceof WritableBinaryObjectInspector) {
BytesWritable bw = (BytesWritable) value;
BytesColumnVector bv = (BytesColumnVector) colVec;
bv.setVal(i, bw.getBytes(), 0, bw.getLength());
} else {
throw new RuntimeException("Unhandled object type " + outputOI.getTypeName() + " inspector class " + outputOI.getClass().getName() + " value class " + value.getClass().getName());
}
}
use of org.apache.hadoop.hive.common.type.HiveDecimal in project hive by apache.
the class RexNodeConverter method convert.
protected RexNode convert(ExprNodeConstantDesc literal) throws CalciteSemanticException {
RexBuilder rexBuilder = cluster.getRexBuilder();
RelDataTypeFactory dtFactory = rexBuilder.getTypeFactory();
PrimitiveTypeInfo hiveType = (PrimitiveTypeInfo) literal.getTypeInfo();
RelDataType calciteDataType = TypeConverter.convert(hiveType, dtFactory);
PrimitiveCategory hiveTypeCategory = hiveType.getPrimitiveCategory();
ConstantObjectInspector coi = literal.getWritableObjectInspector();
Object value = ObjectInspectorUtils.copyToStandardJavaObject(coi.getWritableConstantValue(), coi);
RexNode calciteLiteral = null;
// If value is null, the type should also be VOID.
if (value == null) {
hiveTypeCategory = PrimitiveCategory.VOID;
}
// TODO: Verify if we need to use ConstantObjectInspector to unwrap data
switch(hiveTypeCategory) {
case BOOLEAN:
calciteLiteral = rexBuilder.makeLiteral(((Boolean) value).booleanValue());
break;
case BYTE:
calciteLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Byte) value), calciteDataType);
break;
case SHORT:
calciteLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Short) value), calciteDataType);
break;
case INT:
calciteLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Integer) value));
break;
case LONG:
calciteLiteral = rexBuilder.makeBigintLiteral(new BigDecimal((Long) value));
break;
// TODO: is Decimal an exact numeric or approximate numeric?
case DECIMAL:
if (value instanceof HiveDecimal) {
value = ((HiveDecimal) value).bigDecimalValue();
} else if (value instanceof Decimal128) {
value = ((Decimal128) value).toBigDecimal();
}
if (value == null) {
// literals.
throw new CalciteSemanticException("Expression " + literal.getExprString() + " is not a valid decimal", UnsupportedFeature.Invalid_decimal);
// TODO: return createNullLiteral(literal);
}
BigDecimal bd = (BigDecimal) value;
BigInteger unscaled = bd.unscaledValue();
if (unscaled.compareTo(MIN_LONG_BI) >= 0 && unscaled.compareTo(MAX_LONG_BI) <= 0) {
calciteLiteral = rexBuilder.makeExactLiteral(bd);
} else {
// CBO doesn't support unlimited precision decimals. In practice, this
// will work...
// An alternative would be to throw CboSemanticException and fall back
// to no CBO.
RelDataType relType = cluster.getTypeFactory().createSqlType(SqlTypeName.DECIMAL, unscaled.toString().length(), bd.scale());
calciteLiteral = rexBuilder.makeExactLiteral(bd, relType);
}
break;
case FLOAT:
calciteLiteral = rexBuilder.makeApproxLiteral(new BigDecimal(Float.toString((Float) value)), calciteDataType);
break;
case DOUBLE:
// TODO: The best solution is to support NaN in expression reduction.
if (Double.isNaN((Double) value)) {
throw new CalciteSemanticException("NaN", UnsupportedFeature.Invalid_decimal);
}
calciteLiteral = rexBuilder.makeApproxLiteral(new BigDecimal(Double.toString((Double) value)), calciteDataType);
break;
case CHAR:
if (value instanceof HiveChar) {
value = ((HiveChar) value).getValue();
}
calciteLiteral = rexBuilder.makeCharLiteral(asUnicodeString((String) value));
break;
case VARCHAR:
if (value instanceof HiveVarchar) {
value = ((HiveVarchar) value).getValue();
}
calciteLiteral = rexBuilder.makeCharLiteral(asUnicodeString((String) value));
break;
case STRING:
calciteLiteral = rexBuilder.makeCharLiteral(asUnicodeString((String) value));
break;
case DATE:
Calendar cal = new GregorianCalendar();
cal.setTime((Date) value);
calciteLiteral = rexBuilder.makeDateLiteral(cal);
break;
case TIMESTAMP:
Calendar c = null;
if (value instanceof Calendar) {
c = (Calendar) value;
} else {
c = Calendar.getInstance();
c.setTimeInMillis(((Timestamp) value).getTime());
}
calciteLiteral = rexBuilder.makeTimestampLiteral(c, RelDataType.PRECISION_NOT_SPECIFIED);
break;
case INTERVAL_YEAR_MONTH:
// Calcite year-month literal value is months as BigDecimal
BigDecimal totalMonths = BigDecimal.valueOf(((HiveIntervalYearMonth) value).getTotalMonths());
calciteLiteral = rexBuilder.makeIntervalLiteral(totalMonths, new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
break;
case INTERVAL_DAY_TIME:
// Calcite day-time interval is millis value as BigDecimal
// Seconds converted to millis
BigDecimal secsValueBd = BigDecimal.valueOf(((HiveIntervalDayTime) value).getTotalSeconds() * 1000);
// Nanos converted to millis
BigDecimal nanosValueBd = BigDecimal.valueOf(((HiveIntervalDayTime) value).getNanos(), 6);
calciteLiteral = rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd), new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new SqlParserPos(1, 1)));
break;
case VOID:
calciteLiteral = cluster.getRexBuilder().makeLiteral(null, cluster.getTypeFactory().createSqlType(SqlTypeName.NULL), true);
break;
case BINARY:
case UNKNOWN:
default:
throw new RuntimeException("UnSupported Literal");
}
return calciteLiteral;
}
use of org.apache.hadoop.hive.common.type.HiveDecimal in project hive by apache.
the class ColumnStatsTask method unpackDecimalStats.
private void unpackDecimalStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) {
if (fName.equals("countnulls")) {
long v = ((LongObjectInspector) oi).get(o);
statsObj.getStatsData().getDecimalStats().setNumNulls(v);
} else if (fName.equals("numdistinctvalues")) {
long v = ((LongObjectInspector) oi).get(o);
statsObj.getStatsData().getDecimalStats().setNumDVs(v);
} else if (fName.equals("max")) {
HiveDecimal d = ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o);
statsObj.getStatsData().getDecimalStats().setHighValue(convertToThriftDecimal(d));
} else if (fName.equals("min")) {
HiveDecimal d = ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o);
statsObj.getStatsData().getDecimalStats().setLowValue(convertToThriftDecimal(d));
} else if (fName.equals("ndvbitvector")) {
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
String v = ((StringObjectInspector) poi).getPrimitiveJavaObject(o);
statsObj.getStatsData().getDecimalStats().setBitVectors(v);
;
}
}
Aggregations