Examples with HiveVarchar org.apache.hadoop.hive.common.type.HiveVarchar used on opensource projects

Example 26 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class VerifyFastRow method doVerifyDeserializeRead.

public static void doVerifyDeserializeRead(DeserializeRead deserializeRead, TypeInfo typeInfo, Object object, boolean isNull) throws IOException {
    if (isNull) {
        if (object != null) {
            TestCase.fail("Field reports null but object is not null (class " + object.getClass().getName() + ", " + object.toString() + ")");
        }
        return;
    } else if (object == null) {
        TestCase.fail("Field report not null but object is null");
    }
    switch(typeInfo.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
                switch(primitiveTypeInfo.getPrimitiveCategory()) {
                    case BOOLEAN:
                        {
                            boolean value = deserializeRead.currentBoolean;
                            if (!(object instanceof BooleanWritable)) {
                                TestCase.fail("Boolean expected writable not Boolean");
                            }
                            boolean expected = ((BooleanWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Boolean field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case BYTE:
                        {
                            byte value = deserializeRead.currentByte;
                            if (!(object instanceof ByteWritable)) {
                                TestCase.fail("Byte expected writable not Byte");
                            }
                            byte expected = ((ByteWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Byte field mismatch (expected " + (int) expected + " found " + (int) value + ")");
                            }
                        }
                        break;
                    case SHORT:
                        {
                            short value = deserializeRead.currentShort;
                            if (!(object instanceof ShortWritable)) {
                                TestCase.fail("Short expected writable not Short");
                            }
                            short expected = ((ShortWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Short field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case INT:
                        {
                            int value = deserializeRead.currentInt;
                            if (!(object instanceof IntWritable)) {
                                TestCase.fail("Integer expected writable not Integer");
                            }
                            int expected = ((IntWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Int field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case LONG:
                        {
                            long value = deserializeRead.currentLong;
                            if (!(object instanceof LongWritable)) {
                                TestCase.fail("Long expected writable not Long");
                            }
                            Long expected = ((LongWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Long field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case FLOAT:
                        {
                            float value = deserializeRead.currentFloat;
                            if (!(object instanceof FloatWritable)) {
                                TestCase.fail("Float expected writable not Float");
                            }
                            float expected = ((FloatWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Float field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case DOUBLE:
                        {
                            double value = deserializeRead.currentDouble;
                            if (!(object instanceof DoubleWritable)) {
                                TestCase.fail("Double expected writable not Double");
                            }
                            double expected = ((DoubleWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Double field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case STRING:
                        {
                            byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                            Text text = new Text(stringBytes);
                            String string = text.toString();
                            String expected = ((Text) object).toString();
                            if (!string.equals(expected)) {
                                TestCase.fail("String field mismatch (expected '" + expected + "' found '" + string + "')");
                            }
                        }
                        break;
                    case CHAR:
                        {
                            byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                            Text text = new Text(stringBytes);
                            String string = text.toString();
                            HiveChar hiveChar = new HiveChar(string, ((CharTypeInfo) primitiveTypeInfo).getLength());
                            HiveChar expected = ((HiveCharWritable) object).getHiveChar();
                            if (!hiveChar.equals(expected)) {
                                TestCase.fail("Char field mismatch (expected '" + expected + "' found '" + hiveChar + "')");
                            }
                        }
                        break;
                    case VARCHAR:
                        {
                            byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                            Text text = new Text(stringBytes);
                            String string = text.toString();
                            HiveVarchar hiveVarchar = new HiveVarchar(string, ((VarcharTypeInfo) primitiveTypeInfo).getLength());
                            HiveVarchar expected = ((HiveVarcharWritable) object).getHiveVarchar();
                            if (!hiveVarchar.equals(expected)) {
                                TestCase.fail("Varchar field mismatch (expected '" + expected + "' found '" + hiveVarchar + "')");
                            }
                        }
                        break;
                    case DECIMAL:
                        {
                            HiveDecimal value = deserializeRead.currentHiveDecimalWritable.getHiveDecimal();
                            if (value == null) {
                                TestCase.fail("Decimal field evaluated to NULL");
                            }
                            HiveDecimal expected = ((HiveDecimalWritable) object).getHiveDecimal();
                            if (!value.equals(expected)) {
                                DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
                                int precision = decimalTypeInfo.getPrecision();
                                int scale = decimalTypeInfo.getScale();
                                TestCase.fail("Decimal field mismatch (expected " + expected.toString() + " found " + value.toString() + ") precision " + precision + ", scale " + scale);
                            }
                        }
                        break;
                    case DATE:
                        {
                            Date value = deserializeRead.currentDateWritable.get();
                            Date expected = ((DateWritableV2) object).get();
                            if (!value.equals(expected)) {
                                TestCase.fail("Date field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                            }
                        }
                        break;
                    case TIMESTAMP:
                        {
                            Timestamp value = deserializeRead.currentTimestampWritable.getTimestamp();
                            Timestamp expected = ((TimestampWritableV2) object).getTimestamp();
                            if (!value.equals(expected)) {
                                TestCase.fail("Timestamp field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                            }
                        }
                        break;
                    case INTERVAL_YEAR_MONTH:
                        {
                            HiveIntervalYearMonth value = deserializeRead.currentHiveIntervalYearMonthWritable.getHiveIntervalYearMonth();
                            HiveIntervalYearMonth expected = ((HiveIntervalYearMonthWritable) object).getHiveIntervalYearMonth();
                            if (!value.equals(expected)) {
                                TestCase.fail("HiveIntervalYearMonth field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                            }
                        }
                        break;
                    case INTERVAL_DAY_TIME:
                        {
                            HiveIntervalDayTime value = deserializeRead.currentHiveIntervalDayTimeWritable.getHiveIntervalDayTime();
                            HiveIntervalDayTime expected = ((HiveIntervalDayTimeWritable) object).getHiveIntervalDayTime();
                            if (!value.equals(expected)) {
                                TestCase.fail("HiveIntervalDayTime field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                            }
                        }
                        break;
                    case BINARY:
                        {
                            byte[] byteArray = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                            BytesWritable bytesWritable = (BytesWritable) object;
                            byte[] expected = Arrays.copyOfRange(bytesWritable.getBytes(), 0, bytesWritable.getLength());
                            if (byteArray.length != expected.length) {
                                TestCase.fail("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(byteArray) + ")");
                            }
                            for (int b = 0; b < byteArray.length; b++) {
                                if (byteArray[b] != expected[b]) {
                                    TestCase.fail("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(byteArray) + ")");
                                }
                            }
                        }
                        break;
                    default:
                        throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory());
                }
            }
            break;
        case LIST:
        case MAP:
        case STRUCT:
        case UNION:
            throw new Error("Complex types need to be handled separately");
        default:
            throw new Error("Unknown category " + typeInfo.getCategory());
    }
}

Example 27 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class VectorColumnGroupGenerator method generateRowColumnValue.

private void generateRowColumnValue(int rowIndex, int columnIndex, Random random) {
    GenerateType generateType = generateTypes[columnIndex];
    GenerateCategory category = generateType.getCategory();
    boolean allowNulls = generateType.getAllowNulls();
    if (allowNulls && random.nextInt(100) < 5) {
        isNullArrays[columnIndex][rowIndex] = true;
        return;
    }
    Object array = arrays[columnIndex];
    switch(category) {
        case BOOLEAN:
            {
                boolean value = random.nextBoolean();
                ((boolean[]) array)[rowIndex] = value;
            }
            break;
        case BYTE:
            {
                byte value = (byte) (random.nextBoolean() ? -random.nextInt(-((int) Byte.MIN_VALUE) + 1) : random.nextInt((int) Byte.MAX_VALUE + 1));
                ((byte[]) array)[rowIndex] = value;
            }
            break;
        case SHORT:
            {
                short value = (short) (random.nextBoolean() ? -random.nextInt(-((int) Short.MIN_VALUE) + 1) : random.nextInt((int) Short.MAX_VALUE + 1));
                ((short[]) array)[rowIndex] = value;
            }
            break;
        case INT:
            {
                int value = random.nextInt();
                ((int[]) array)[rowIndex] = value;
            }
            break;
        case LONG:
            {
                long value = random.nextLong();
                ((long[]) array)[rowIndex] = value;
            }
            break;
        case FLOAT:
            {
                float value = random.nextLong();
                ((float[]) array)[rowIndex] = value;
            }
            break;
        case DOUBLE:
            {
                double value = random.nextLong();
                ((double[]) array)[rowIndex] = value;
            }
            break;
        case STRING:
            {
                String value = RandomTypeUtil.getRandString(random);
                ((String[]) array)[rowIndex] = value;
            }
            break;
        case BINARY:
            {
                byte[] value = RandomTypeUtil.getRandBinary(random, 10);
                ((byte[][]) array)[rowIndex] = value;
            }
            break;
        case DATE:
            {
                Date value = RandomTypeUtil.getRandDate(random);
                ((Date[]) array)[rowIndex] = value;
            }
            break;
        case TIMESTAMP:
            {
                Timestamp value = RandomTypeUtil.getRandTimestamp(random).toSqlTimestamp();
                ((Timestamp[]) array)[rowIndex] = value;
            }
            break;
        case CHAR:
            {
                // UNDONE: Use CharTypeInfo.maxLength
                HiveChar value = new HiveChar(RandomTypeUtil.getRandString(random), 10);
                ((HiveChar[]) array)[rowIndex] = value;
            }
            break;
        case VARCHAR:
            {
                // UNDONE: Use VarcharTypeInfo.maxLength
                HiveVarchar value = new HiveVarchar(RandomTypeUtil.getRandString(random), 10);
                ((HiveVarchar[]) array)[rowIndex] = value;
            }
            break;
        case DECIMAL:
            {
                HiveDecimalWritable value = new HiveDecimalWritable(RandomTypeUtil.getRandHiveDecimal(random));
                ((HiveDecimalWritable[]) array)[rowIndex] = value;
            }
            break;
        case LIST:
        case MAP:
        case STRUCT:
        case UNION:
        default:
    }
}

Example 28 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class VectorColumnGroupGenerator method fillDownRowColumnValue.

private void fillDownRowColumnValue(int rowIndex, int columnIndex, int seriesCount, Random random) {
    GenerateType generateType = generateTypes[columnIndex];
    GenerateCategory category = generateType.getCategory();
    boolean allowNulls = generateType.getAllowNulls();
    Object array = arrays[columnIndex];
    boolean[] isNull = isNullArrays[columnIndex];
    if (allowNulls && isNull[rowIndex]) {
        for (int i = 1; i < seriesCount; i++) {
            isNull[rowIndex + i] = true;
        }
        return;
    }
    switch(category) {
        case BOOLEAN:
            {
                boolean[] booleanArray = ((boolean[]) array);
                boolean value = booleanArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    booleanArray[rowIndex + i] = value;
                }
            }
            break;
        case BYTE:
            {
                byte[] byteArray = ((byte[]) array);
                byte value = byteArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    byteArray[rowIndex + i] = value;
                }
            }
            break;
        case SHORT:
            {
                short[] shortArray = ((short[]) array);
                short value = shortArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    shortArray[rowIndex + i] = value;
                }
            }
            break;
        case INT:
            {
                int[] intArray = ((int[]) array);
                int value = intArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    intArray[rowIndex + i] = value;
                }
            }
            break;
        case LONG:
            {
                long[] longArray = ((long[]) array);
                long value = longArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    longArray[rowIndex + i] = value;
                }
            }
            break;
        case FLOAT:
            {
                float[] floatArray = ((float[]) array);
                float value = floatArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    floatArray[rowIndex + i] = value;
                }
            }
            break;
        case DOUBLE:
            {
                double[] doubleArray = ((double[]) array);
                double value = doubleArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    doubleArray[rowIndex + i] = value;
                }
            }
            break;
        case STRING:
            {
                String[] stringArray = ((String[]) array);
                String value = stringArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    stringArray[rowIndex + i] = value;
                }
            }
            break;
        case BINARY:
            {
                byte[][] byteArrayArray = ((byte[][]) array);
                byte[] value = byteArrayArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    byteArrayArray[rowIndex + i] = value;
                }
            }
            break;
        case DATE:
            {
                Date[] dateArray = ((Date[]) array);
                Date value = dateArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    dateArray[rowIndex + i] = value;
                }
            }
            break;
        case TIMESTAMP:
            {
                Timestamp[] timestampArray = ((Timestamp[]) array);
                Timestamp value = timestampArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    timestampArray[rowIndex + i] = value;
                }
            }
            break;
        case CHAR:
            {
                HiveChar[] hiveCharArray = ((HiveChar[]) array);
                HiveChar value = hiveCharArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    hiveCharArray[rowIndex + i] = value;
                }
            }
            break;
        case VARCHAR:
            {
                HiveVarchar[] hiveVarcharArray = ((HiveVarchar[]) array);
                HiveVarchar value = hiveVarcharArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    hiveVarcharArray[rowIndex + i] = value;
                }
            }
            break;
        case DECIMAL:
            {
                HiveDecimalWritable[] hiveDecimalWritableArray = ((HiveDecimalWritable[]) array);
                HiveDecimalWritable value = hiveDecimalWritableArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    hiveDecimalWritableArray[rowIndex + i] = value;
                }
            }
            break;
        case LIST:
        case MAP:
        case STRUCT:
        case UNION:
        default:
    }
}

Example 29 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class TestParquetFilterPredicate method testFilterComplexTypes.

/**
 * Check the converted filter predicate is null if unsupported types are included
 * @throws Exception
 */
@Test
public void testFilterComplexTypes() throws Exception {
    SearchArgument sarg = SearchArgumentFactory.newBuilder().startAnd().lessThan("x", PredicateLeaf.Type.DATE, Date.valueOf("1970-1-11")).lessThanEquals("y", PredicateLeaf.Type.STRING, new HiveChar("hi", 10).toString()).equals("z", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("1.0")).end().build();
    MessageType schema = MessageTypeParser.parseMessageType("message test {" + " required int32 x; required binary y; required binary z;}");
    Map<String, TypeInfo> columnTypes = new HashMap<>();
    columnTypes.put("x", TypeInfoFactory.getPrimitiveTypeInfo("date"));
    columnTypes.put("y", TypeInfoFactory.getCharTypeInfo(10));
    columnTypes.put("z", TypeInfoFactory.getDecimalTypeInfo(4, 2));
    assertEquals(null, ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema, columnTypes));
    sarg = SearchArgumentFactory.newBuilder().startNot().startOr().isNull("x", PredicateLeaf.Type.LONG).between("y", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("10"), new HiveDecimalWritable("20.0")).in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L).nullSafeEquals("a", PredicateLeaf.Type.STRING, new HiveVarchar("stinger", 100).toString()).end().end().build();
    schema = MessageTypeParser.parseMessageType("message test {" + " optional int32 x; required binary y; required int32 z;" + " optional binary a;}");
    columnTypes = new HashMap<>();
    columnTypes.put("x", TypeInfoFactory.getPrimitiveTypeInfo("int"));
    columnTypes.put("y", TypeInfoFactory.getDecimalTypeInfo(4, 2));
    columnTypes.put("z", TypeInfoFactory.getPrimitiveTypeInfo("int"));
    columnTypes.put("z", TypeInfoFactory.getCharTypeInfo(100));
    assertEquals(null, ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema, columnTypes));
}

Example 30 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class TestParquetFilterPredicate method testFilterComplexTypes2.

/**
 * Check the converted filter predicate is null if unsupported types are included
 * @throws Exception
 */
@Test
public void testFilterComplexTypes2() throws Exception {
    SearchArgument sarg = SearchArgumentFactory.newBuilder().startAnd().lessThan("x", PredicateLeaf.Type.DATE, Date.valueOf("2005-3-12")).lessThanEquals("y", PredicateLeaf.Type.STRING, new HiveChar("hi", 10).toString()).equals("z", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("1.0")).end().build();
    MessageType schema = MessageTypeParser.parseMessageType("message test {" + " required int32 x; required binary y; required binary z;}");
    Map<String, TypeInfo> columnTypes = new HashMap<>();
    columnTypes.put("x", TypeInfoFactory.getPrimitiveTypeInfo("date"));
    columnTypes.put("y", TypeInfoFactory.getCharTypeInfo(10));
    columnTypes.put("z", TypeInfoFactory.getDecimalTypeInfo(4, 2));
    assertEquals(null, ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema, columnTypes));
    sarg = SearchArgumentFactory.newBuilder().startNot().startOr().isNull("x", PredicateLeaf.Type.LONG).between("y", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("10"), new HiveDecimalWritable("20.0")).in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L).nullSafeEquals("a", PredicateLeaf.Type.STRING, new HiveVarchar("stinger", 100).toString()).end().end().build();
    schema = MessageTypeParser.parseMessageType("message test {" + " optional int32 x; required binary y; required int32 z;" + " optional binary a;}");
    columnTypes = new HashMap<>();
    columnTypes.put("x", TypeInfoFactory.getPrimitiveTypeInfo("int"));
    columnTypes.put("y", TypeInfoFactory.getDecimalTypeInfo(4, 2));
    columnTypes.put("z", TypeInfoFactory.getPrimitiveTypeInfo("int"));
    columnTypes.put("a", TypeInfoFactory.getCharTypeInfo(100));
    assertEquals(null, ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema, columnTypes));
}