Examples with HiveVarchar org.apache.hadoop.hive.common.type.HiveVarchar used on opensource projects

Example 16 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class TestVectorSerDeRow method getObjectDisplayString.

private String getObjectDisplayString(Object object) {
    StringBuilder sb = new StringBuilder();
    if (object == null) {
        sb.append("NULL");
    } else if (object instanceof Text || object instanceof HiveChar || object instanceof HiveCharWritable || object instanceof HiveVarchar || object instanceof HiveVarcharWritable) {
        final String string;
        if (object instanceof Text) {
            Text text = (Text) object;
            string = text.toString();
        } else if (object instanceof HiveChar) {
            HiveChar hiveChar = (HiveChar) object;
            string = hiveChar.getStrippedValue();
        } else if (object instanceof HiveCharWritable) {
            HiveChar hiveChar = ((HiveCharWritable) object).getHiveChar();
            string = hiveChar.getStrippedValue();
        } else if (object instanceof HiveVarchar) {
            HiveVarchar hiveVarchar = (HiveVarchar) object;
            string = hiveVarchar.getValue();
        } else if (object instanceof HiveVarcharWritable) {
            HiveVarchar hiveVarchar = ((HiveVarcharWritable) object).getHiveVarchar();
            string = hiveVarchar.getValue();
        } else {
            throw new RuntimeException("Unexpected");
        }
        byte[] bytes = string.getBytes();
        final int byteLength = bytes.length;
        sb.append("'");
        sb.append(string);
        sb.append("' (byte length ");
        sb.append(bytes.length);
        sb.append(", string length ");
        sb.append(string.length());
        sb.append(", bytes ");
        sb.append(VectorizedBatchUtil.displayBytes(bytes, 0, byteLength));
        sb.append(")");
    } else {
        sb.append(object.toString());
    }
    return sb.toString();
}

Example 17 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class RexNodeConverter method convert.

protected RexNode convert(ExprNodeConstantDesc literal) throws CalciteSemanticException {
    final RelDataTypeFactory dtFactory = rexBuilder.getTypeFactory();
    final PrimitiveTypeInfo hiveType = (PrimitiveTypeInfo) literal.getTypeInfo();
    final RelDataType calciteDataType = TypeConverter.convert(hiveType, dtFactory);
    PrimitiveCategory hiveTypeCategory = hiveType.getPrimitiveCategory();
    ConstantObjectInspector coi = literal.getWritableObjectInspector();
    Object value = ObjectInspectorUtils.copyToStandardJavaObject(coi.getWritableConstantValue(), coi);
    RexNode calciteLiteral = null;
    // If value is null, the type should also be VOID.
    if (value == null) {
        hiveTypeCategory = PrimitiveCategory.VOID;
    }
    // TODO: Verify if we need to use ConstantObjectInspector to unwrap data
    switch(hiveTypeCategory) {
        case BOOLEAN:
            calciteLiteral = rexBuilder.makeLiteral(((Boolean) value).booleanValue());
            break;
        case BYTE:
            calciteLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Byte) value), calciteDataType);
            break;
        case SHORT:
            calciteLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Short) value), calciteDataType);
            break;
        case INT:
            calciteLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Integer) value));
            break;
        case LONG:
            calciteLiteral = rexBuilder.makeBigintLiteral(new BigDecimal((Long) value));
            break;
        case DECIMAL:
            if (value instanceof HiveDecimal) {
                value = ((HiveDecimal) value).bigDecimalValue();
            } else if (value instanceof Decimal128) {
                value = ((Decimal128) value).toBigDecimal();
            }
            if (value == null) {
                // literals.
                throw new CalciteSemanticException("Expression " + literal.getExprString() + " is not a valid decimal", UnsupportedFeature.Invalid_decimal);
            // TODO: return createNullLiteral(literal);
            }
            calciteLiteral = rexBuilder.makeExactLiteral((BigDecimal) value, calciteDataType);
            break;
        case FLOAT:
            calciteLiteral = rexBuilder.makeApproxLiteral(new BigDecimal(Float.toString((Float) value)), calciteDataType);
            break;
        case DOUBLE:
            // TODO: The best solution is to support NaN in expression reduction.
            if (Double.isNaN((Double) value)) {
                throw new CalciteSemanticException("NaN", UnsupportedFeature.Invalid_decimal);
            }
            calciteLiteral = rexBuilder.makeApproxLiteral(new BigDecimal(Double.toString((Double) value)), calciteDataType);
            break;
        case CHAR:
            if (value instanceof HiveChar) {
                value = ((HiveChar) value).getValue();
            }
            final int lengthChar = TypeInfoUtils.getCharacterLengthForType(hiveType);
            RelDataType charType = rexBuilder.getTypeFactory().createTypeWithCharsetAndCollation(rexBuilder.getTypeFactory().createSqlType(SqlTypeName.CHAR, lengthChar), Charset.forName(ConversionUtil.NATIVE_UTF16_CHARSET_NAME), SqlCollation.IMPLICIT);
            calciteLiteral = rexBuilder.makeLiteral(RexNodeExprFactory.makeHiveUnicodeString((String) value), charType, false);
            break;
        case VARCHAR:
            if (value instanceof HiveVarchar) {
                value = ((HiveVarchar) value).getValue();
            }
            final int lengthVarchar = TypeInfoUtils.getCharacterLengthForType(hiveType);
            RelDataType varcharType = rexBuilder.getTypeFactory().createTypeWithCharsetAndCollation(rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR, lengthVarchar), Charset.forName(ConversionUtil.NATIVE_UTF16_CHARSET_NAME), SqlCollation.IMPLICIT);
            calciteLiteral = rexBuilder.makeLiteral(RexNodeExprFactory.makeHiveUnicodeString((String) value), varcharType, true);
            break;
        case STRING:
            RelDataType stringType = rexBuilder.getTypeFactory().createTypeWithCharsetAndCollation(rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR, Integer.MAX_VALUE), Charset.forName(ConversionUtil.NATIVE_UTF16_CHARSET_NAME), SqlCollation.IMPLICIT);
            calciteLiteral = rexBuilder.makeLiteral(RexNodeExprFactory.makeHiveUnicodeString((String) value), stringType, true);
            break;
        case DATE:
            final Date date = (Date) value;
            calciteLiteral = rexBuilder.makeDateLiteral(DateString.fromDaysSinceEpoch(date.toEpochDay()));
            break;
        case TIMESTAMP:
            final TimestampString tsString;
            if (value instanceof Calendar) {
                tsString = TimestampString.fromCalendarFields((Calendar) value);
            } else {
                final Timestamp ts = (Timestamp) value;
                tsString = TimestampString.fromMillisSinceEpoch(ts.toEpochMilli()).withNanos(ts.getNanos());
            }
            // Must call makeLiteral, not makeTimestampLiteral
            // to have the RexBuilder.roundTime logic kick in
            calciteLiteral = rexBuilder.makeLiteral(tsString, rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TIMESTAMP, rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP)), false);
            break;
        case TIMESTAMPLOCALTZ:
            final TimestampString tsLocalTZString;
            Instant i = ((TimestampTZ) value).getZonedDateTime().toInstant();
            tsLocalTZString = TimestampString.fromMillisSinceEpoch(i.toEpochMilli()).withNanos(i.getNano());
            calciteLiteral = rexBuilder.makeTimestampWithLocalTimeZoneLiteral(tsLocalTZString, rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE));
            break;
        case INTERVAL_YEAR_MONTH:
            // Calcite year-month literal value is months as BigDecimal
            BigDecimal totalMonths = BigDecimal.valueOf(((HiveIntervalYearMonth) value).getTotalMonths());
            calciteLiteral = rexBuilder.makeIntervalLiteral(totalMonths, new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
            break;
        case INTERVAL_DAY_TIME:
            // Calcite day-time interval is millis value as BigDecimal
            // Seconds converted to millis
            BigDecimal secsValueBd = BigDecimal.valueOf(((HiveIntervalDayTime) value).getTotalSeconds() * 1000);
            // Nanos converted to millis
            BigDecimal nanosValueBd = BigDecimal.valueOf(((HiveIntervalDayTime) value).getNanos(), 6);
            calciteLiteral = rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd), new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new SqlParserPos(1, 1)));
            break;
        case VOID:
            calciteLiteral = rexBuilder.makeLiteral(null, calciteDataType, true);
            break;
        case BINARY:
        case UNKNOWN:
        default:
            throw new RuntimeException("Unsupported Literal");
    }
    return calciteLiteral;
}

Example 18 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class TestVectorStringExpressions method testStringColCompareVarCharScalarProjection.

@Test
public void testStringColCompareVarCharScalarProjection() throws HiveException {
    VectorizedRowBatch batch = makeStringBatch();
    VectorExpression expr;
    expr = new StringGroupColEqualVarCharScalar(0, new HiveVarchar(new String(red2), 8).getValue().getBytes(), 2);
    expr.evaluate(batch);
    Assert.assertEquals(3, batch.size);
    LongColumnVector outVector = (LongColumnVector) batch.cols[2];
    Assert.assertEquals(1, outVector.vector[0]);
    Assert.assertEquals(0, outVector.vector[1]);
    Assert.assertEquals(0, outVector.vector[2]);
    batch = makeStringBatch();
    expr = new StringGroupColEqualVarCharScalar(0, new HiveVarchar(new String(green), 10).getValue().getBytes(), 2);
    expr.evaluate(batch);
    Assert.assertEquals(3, batch.size);
    outVector = (LongColumnVector) batch.cols[2];
    Assert.assertEquals(0, outVector.vector[0]);
    Assert.assertEquals(1, outVector.vector[1]);
    Assert.assertEquals(0, outVector.vector[2]);
}

Example 19 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class TestVectorStringExpressions method testColConcatVarCharScalar.

@Test
public void testColConcatVarCharScalar() throws HiveException {
    // has nulls, not repeating
    VectorizedRowBatch batch = makeStringBatch();
    StringGroupColConcatStringScalar expr = new StringGroupColConcatStringScalar(0, new HiveVarchar(new String(red), 14).getValue().getBytes(), 1);
    expr.evaluate(batch);
    BytesColumnVector outCol = (BytesColumnVector) batch.cols[1];
    int cmp = StringExpr.compare(redred, 0, redred.length, outCol.vector[0], outCol.start[0], outCol.length[0]);
    Assert.assertEquals(0, cmp);
    Assert.assertTrue(outCol.isNull[2]);
    int cmp2 = StringExpr.compare(greenred, 0, greenred.length, outCol.vector[1], outCol.start[1], outCol.length[1]);
    Assert.assertEquals(0, cmp2);
    Assert.assertFalse(outCol.noNulls);
    Assert.assertFalse(outCol.isRepeating);
    // no nulls, not repeating
    batch = makeStringBatch();
    batch.cols[0].noNulls = true;
    expr.evaluate(batch);
    outCol = (BytesColumnVector) batch.cols[1];
    cmp = StringExpr.compare(redred, 0, redred.length, outCol.vector[0], outCol.start[0], outCol.length[0]);
    Assert.assertEquals(0, cmp);
    cmp2 = StringExpr.compare(greenred, 0, greenred.length, outCol.vector[1], outCol.start[1], outCol.length[1]);
    Assert.assertEquals(0, cmp2);
    int cmp3 = StringExpr.compare(red, 0, red.length, outCol.vector[2], outCol.start[2], outCol.length[2]);
    Assert.assertEquals(0, cmp3);
    Assert.assertTrue(outCol.noNulls);
    Assert.assertFalse(outCol.isRepeating);
    // has nulls, is repeating
    batch = makeStringBatch();
    batch.cols[0].isRepeating = true;
    expr.evaluate(batch);
    outCol = (BytesColumnVector) batch.cols[1];
    cmp = StringExpr.compare(redred, 0, redred.length, outCol.vector[0], outCol.start[0], outCol.length[0]);
    Assert.assertEquals(0, cmp);
    Assert.assertTrue(outCol.isRepeating);
    // no nulls, is repeating
    batch = makeStringBatch();
    batch.cols[0].isRepeating = true;
    batch.cols[0].noNulls = true;
    expr.evaluate(batch);
    outCol = (BytesColumnVector) batch.cols[1];
    cmp = StringExpr.compare(redred, 0, redred.length, outCol.vector[0], outCol.start[0], outCol.length[0]);
    Assert.assertEquals(0, cmp);
    Assert.assertTrue(outCol.isRepeating);
    Assert.assertTrue(outCol.noNulls);
}

Example 20 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class TestVectorStringExpressions method testStringColCompareVarCharScalarFilter.

@Test
public // Test string column to VARCHAR literal comparison
void testStringColCompareVarCharScalarFilter() throws HiveException {
    VectorizedRowBatch batch = makeStringBatch();
    VectorExpression expr;
    expr = new FilterStringGroupColEqualVarCharScalar(0, new HiveVarchar(new String(red2), 10).getValue().getBytes());
    expr.evaluate(batch);
    // only red qualifies, and it's in entry 0
    Assert.assertTrue(batch.size == 1);
    Assert.assertTrue(batch.selected[0] == 0);
    batch = makeStringBatch();
    expr = new FilterStringGroupColLessVarCharScalar(0, new HiveVarchar(new String(red2), 8).getValue().getBytes());
    expr.evaluate(batch);
    // only green qualifies, and it's in entry 1
    Assert.assertTrue(batch.size == 1);
    Assert.assertTrue(batch.selected[0] == 1);
    batch = makeStringBatch();
    expr = new FilterStringGroupColGreaterEqualVarCharScalar(0, new HiveVarchar(new String(green), 12).getValue().getBytes());
    expr.evaluate(batch);
    // green and red qualify
    Assert.assertTrue(batch.size == 2);
    Assert.assertTrue(batch.selected[0] == 0);
    Assert.assertTrue(batch.selected[1] == 1);
}