use of org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo in project hive by apache.
the class TestHiveAccumuloTypes method testBinaryTypes.
@Test
public void testBinaryTypes() throws Exception {
final String tableName = test.getMethodName(), user = "root", pass = "";
MockInstance mockInstance = new MockInstance(test.getMethodName());
Connector conn = mockInstance.getConnector(user, new PasswordToken(pass));
HiveAccumuloTableInputFormat inputformat = new HiveAccumuloTableInputFormat();
JobConf conf = new JobConf();
conf.set(AccumuloSerDeParameters.TABLE_NAME, tableName);
conf.set(AccumuloSerDeParameters.USE_MOCK_INSTANCE, "true");
conf.set(AccumuloSerDeParameters.INSTANCE_NAME, test.getMethodName());
conf.set(AccumuloSerDeParameters.USER_NAME, user);
conf.set(AccumuloSerDeParameters.USER_PASS, pass);
// not used for mock, but
conf.set(AccumuloSerDeParameters.ZOOKEEPERS, "localhost:2181");
// required by input format.
conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS, AccumuloHiveConstants.ROWID + ",cf:string,cf:boolean,cf:tinyint,cf:smallint,cf:int,cf:bigint" + ",cf:float,cf:double,cf:decimal,cf:date,cf:timestamp,cf:char,cf:varchar");
conf.set(serdeConstants.LIST_COLUMNS, "string,string,boolean,tinyint,smallint,int,bigint,float,double,decimal,date,timestamp,char(4),varchar(7)");
conf.set(serdeConstants.LIST_COLUMN_TYPES, "string,string,boolean,tinyint,smallint,int,bigint,float,double,decimal,date,timestamp,char(4),varchar(7)");
conf.set(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE, "binary");
conn.tableOperations().create(tableName);
BatchWriterConfig writerConf = new BatchWriterConfig();
BatchWriter writer = conn.createBatchWriter(tableName, writerConf);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(baos);
String cf = "cf";
byte[] cfBytes = cf.getBytes();
Mutation m = new Mutation("row1");
// string
String stringValue = "string";
JavaStringObjectInspector stringOI = (JavaStringObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME));
LazyUtils.writePrimitiveUTF8(baos, stringOI.create(stringValue), stringOI, false, (byte) 0, null);
m.put(cfBytes, "string".getBytes(), baos.toByteArray());
// boolean
boolean booleanValue = true;
baos.reset();
JavaBooleanObjectInspector booleanOI = (JavaBooleanObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME));
LazyUtils.writePrimitive(baos, booleanOI.create(booleanValue), booleanOI);
m.put(cfBytes, "boolean".getBytes(), baos.toByteArray());
// tinyint
byte tinyintValue = -127;
baos.reset();
JavaByteObjectInspector byteOI = (JavaByteObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TINYINT_TYPE_NAME));
LazyUtils.writePrimitive(baos, tinyintValue, byteOI);
m.put(cfBytes, "tinyint".getBytes(), baos.toByteArray());
// smallint
short smallintValue = Short.MAX_VALUE;
baos.reset();
JavaShortObjectInspector shortOI = (JavaShortObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.SMALLINT_TYPE_NAME));
LazyUtils.writePrimitive(baos, smallintValue, shortOI);
m.put(cfBytes, "smallint".getBytes(), baos.toByteArray());
// int
int intValue = Integer.MAX_VALUE;
baos.reset();
JavaIntObjectInspector intOI = (JavaIntObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME));
LazyUtils.writePrimitive(baos, intValue, intOI);
m.put(cfBytes, "int".getBytes(), baos.toByteArray());
// bigint
long bigintValue = Long.MAX_VALUE;
baos.reset();
JavaLongObjectInspector longOI = (JavaLongObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME));
LazyUtils.writePrimitive(baos, bigintValue, longOI);
m.put(cfBytes, "bigint".getBytes(), baos.toByteArray());
// float
float floatValue = Float.MAX_VALUE;
baos.reset();
JavaFloatObjectInspector floatOI = (JavaFloatObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME));
LazyUtils.writePrimitive(baos, floatValue, floatOI);
m.put(cfBytes, "float".getBytes(), baos.toByteArray());
// double
double doubleValue = Double.MAX_VALUE;
baos.reset();
JavaDoubleObjectInspector doubleOI = (JavaDoubleObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME));
LazyUtils.writePrimitive(baos, doubleValue, doubleOI);
m.put(cfBytes, "double".getBytes(), baos.toByteArray());
// decimal
baos.reset();
HiveDecimal decimalValue = HiveDecimal.create(65536l);
HiveDecimalWritable decimalWritable = new HiveDecimalWritable(decimalValue);
decimalWritable.write(out);
m.put(cfBytes, "decimal".getBytes(), baos.toByteArray());
// date
baos.reset();
Date now = new Date(System.currentTimeMillis());
DateWritable dateWritable = new DateWritable(now);
Date dateValue = dateWritable.get();
dateWritable.write(out);
m.put(cfBytes, "date".getBytes(), baos.toByteArray());
// tiemestamp
baos.reset();
Timestamp timestampValue = new Timestamp(now.getTime());
ByteStream.Output output = new ByteStream.Output();
TimestampWritable timestampWritable = new TimestampWritable(new Timestamp(now.getTime()));
timestampWritable.write(new DataOutputStream(output));
output.close();
m.put(cfBytes, "timestamp".getBytes(), output.toByteArray());
// char
baos.reset();
HiveChar charValue = new HiveChar("char", 4);
JavaHiveCharObjectInspector charOI = (JavaHiveCharObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(new CharTypeInfo(4));
LazyUtils.writePrimitiveUTF8(baos, charOI.create(charValue), charOI, false, (byte) 0, null);
m.put(cfBytes, "char".getBytes(), baos.toByteArray());
baos.reset();
HiveVarchar varcharValue = new HiveVarchar("varchar", 7);
JavaHiveVarcharObjectInspector varcharOI = (JavaHiveVarcharObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(new VarcharTypeInfo(7));
LazyUtils.writePrimitiveUTF8(baos, varcharOI.create(varcharValue), varcharOI, false, (byte) 0, null);
m.put(cfBytes, "varchar".getBytes(), baos.toByteArray());
writer.addMutation(m);
writer.close();
for (Entry<Key, Value> e : conn.createScanner(tableName, new Authorizations())) {
System.out.println(e);
}
// Create the RecordReader
FileInputFormat.addInputPath(conf, new Path("unused"));
InputSplit[] splits = inputformat.getSplits(conf, 0);
assertEquals(splits.length, 1);
RecordReader<Text, AccumuloHiveRow> reader = inputformat.getRecordReader(splits[0], conf, null);
Text key = reader.createKey();
AccumuloHiveRow value = reader.createValue();
reader.next(key, value);
Assert.assertEquals(13, value.getTuples().size());
ByteArrayRef byteRef = new ByteArrayRef();
// string
Text cfText = new Text(cf), cqHolder = new Text();
cqHolder.set("string");
byte[] valueBytes = value.getValue(cfText, cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyStringObjectInspector lazyStringOI = LazyPrimitiveObjectInspectorFactory.getLazyStringObjectInspector(false, (byte) 0);
LazyString lazyString = (LazyString) LazyFactory.createLazyObject(lazyStringOI);
lazyString.init(byteRef, 0, valueBytes.length);
Assert.assertEquals(stringValue, lazyString.getWritableObject().toString());
// boolean
cqHolder.set("boolean");
valueBytes = value.getValue(cfText, cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyBooleanObjectInspector lazyBooleanOI = (LazyBooleanObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME));
LazyBoolean lazyBoolean = (LazyBoolean) LazyFactory.createLazyPrimitiveBinaryClass(lazyBooleanOI);
lazyBoolean.init(byteRef, 0, valueBytes.length);
Assert.assertEquals(booleanValue, lazyBoolean.getWritableObject().get());
// tinyint
cqHolder.set("tinyint");
valueBytes = value.getValue(cfText, cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyByteObjectInspector lazyByteOI = (LazyByteObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TINYINT_TYPE_NAME));
LazyByte lazyByte = (LazyByte) LazyFactory.createLazyPrimitiveBinaryClass(lazyByteOI);
lazyByte.init(byteRef, 0, valueBytes.length);
Assert.assertEquals(tinyintValue, lazyByte.getWritableObject().get());
// smallint
cqHolder.set("smallint");
valueBytes = value.getValue(cfText, cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyShortObjectInspector lazyShortOI = (LazyShortObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.SMALLINT_TYPE_NAME));
LazyShort lazyShort = (LazyShort) LazyFactory.createLazyPrimitiveBinaryClass(lazyShortOI);
lazyShort.init(byteRef, 0, valueBytes.length);
Assert.assertEquals(smallintValue, lazyShort.getWritableObject().get());
// int
cqHolder.set("int");
valueBytes = value.getValue(cfText, cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyIntObjectInspector lazyIntOI = (LazyIntObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME));
LazyInteger lazyInt = (LazyInteger) LazyFactory.createLazyPrimitiveBinaryClass(lazyIntOI);
lazyInt.init(byteRef, 0, valueBytes.length);
Assert.assertEquals(intValue, lazyInt.getWritableObject().get());
// bigint
cqHolder.set("bigint");
valueBytes = value.getValue(cfText, cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyLongObjectInspector lazyLongOI = (LazyLongObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME));
LazyLong lazyLong = (LazyLong) LazyFactory.createLazyPrimitiveBinaryClass(lazyLongOI);
lazyLong.init(byteRef, 0, valueBytes.length);
Assert.assertEquals(bigintValue, lazyLong.getWritableObject().get());
// float
cqHolder.set("float");
valueBytes = value.getValue(cfText, cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyFloatObjectInspector lazyFloatOI = (LazyFloatObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME));
LazyFloat lazyFloat = (LazyFloat) LazyFactory.createLazyPrimitiveBinaryClass(lazyFloatOI);
lazyFloat.init(byteRef, 0, valueBytes.length);
Assert.assertEquals(floatValue, lazyFloat.getWritableObject().get(), 0);
// double
cqHolder.set("double");
valueBytes = value.getValue(cfText, cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyDoubleObjectInspector lazyDoubleOI = (LazyDoubleObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME));
LazyDouble lazyDouble = (LazyDouble) LazyFactory.createLazyPrimitiveBinaryClass(lazyDoubleOI);
lazyDouble.init(byteRef, 0, valueBytes.length);
Assert.assertEquals(doubleValue, lazyDouble.getWritableObject().get(), 0);
// decimal
cqHolder.set("decimal");
valueBytes = value.getValue(cfText, cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
ByteArrayInputStream bais = new ByteArrayInputStream(valueBytes);
DataInputStream in = new DataInputStream(bais);
decimalWritable.readFields(in);
Assert.assertEquals(decimalValue, decimalWritable.getHiveDecimal());
// date
cqHolder.set("date");
valueBytes = value.getValue(cfText, cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
bais = new ByteArrayInputStream(valueBytes);
in = new DataInputStream(bais);
dateWritable.readFields(in);
Assert.assertEquals(dateValue, dateWritable.get());
// timestamp
cqHolder.set("timestamp");
valueBytes = value.getValue(cfText, cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
bais = new ByteArrayInputStream(valueBytes);
in = new DataInputStream(bais);
timestampWritable.readFields(in);
Assert.assertEquals(timestampValue, timestampWritable.getTimestamp());
// char
cqHolder.set("char");
valueBytes = value.getValue(cfText, cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyHiveCharObjectInspector lazyCharOI = (LazyHiveCharObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(new CharTypeInfo(4));
LazyHiveChar lazyChar = (LazyHiveChar) LazyFactory.createLazyObject(lazyCharOI);
lazyChar.init(byteRef, 0, valueBytes.length);
Assert.assertEquals(charValue, lazyChar.getWritableObject().getHiveChar());
// varchar
cqHolder.set("varchar");
valueBytes = value.getValue(cfText, cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyHiveVarcharObjectInspector lazyVarcharOI = (LazyHiveVarcharObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(new VarcharTypeInfo(7));
LazyHiveVarchar lazyVarchar = (LazyHiveVarchar) LazyFactory.createLazyObject(lazyVarcharOI);
lazyVarchar.init(byteRef, 0, valueBytes.length);
Assert.assertEquals(varcharValue.toString(), lazyVarchar.getWritableObject().getHiveVarchar().toString());
}
use of org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo in project hive by apache.
the class HCatBaseStorer method getJavaObj.
/**
* Convert from Pig value object to Hive value object
* This method assumes that {@link #validateSchema(org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema, org.apache.hive.hcatalog.data.schema.HCatFieldSchema, org.apache.pig.impl.logicalLayer.schema.Schema, org.apache.hive.hcatalog.data.schema.HCatSchema, int)}
* which checks the types in Pig schema are compatible with target Hive table, has been called.
*/
private Object getJavaObj(Object pigObj, HCatFieldSchema hcatFS) throws HCatException, BackendException {
try {
if (pigObj == null)
return null;
// The real work-horse. Spend time and energy in this method if there is
// need to keep HCatStorer lean and go fast.
Type type = hcatFS.getType();
switch(type) {
case BINARY:
return ((DataByteArray) pigObj).get();
case STRUCT:
HCatSchema structSubSchema = hcatFS.getStructSubSchema();
// Unwrap the tuple.
List<Object> all = ((Tuple) pigObj).getAll();
ArrayList<Object> converted = new ArrayList<Object>(all.size());
for (int i = 0; i < all.size(); i++) {
converted.add(getJavaObj(all.get(i), structSubSchema.get(i)));
}
return converted;
case ARRAY:
// Unwrap the bag.
DataBag pigBag = (DataBag) pigObj;
HCatFieldSchema tupFS = hcatFS.getArrayElementSchema().get(0);
boolean needTuple = tupFS.getType() == Type.STRUCT;
List<Object> bagContents = new ArrayList<Object>((int) pigBag.size());
Iterator<Tuple> bagItr = pigBag.iterator();
while (bagItr.hasNext()) {
// If there is only one element in tuple contained in bag, we throw away the tuple.
bagContents.add(getJavaObj(needTuple ? bagItr.next() : bagItr.next().get(0), tupFS));
}
return bagContents;
case MAP:
Map<?, ?> pigMap = (Map<?, ?>) pigObj;
Map<Object, Object> typeMap = new HashMap<Object, Object>();
for (Entry<?, ?> entry : pigMap.entrySet()) {
// the value has a schema and not a FieldSchema
typeMap.put(// Schema validation enforces that the Key is a String
(String) entry.getKey(), getJavaObj(entry.getValue(), hcatFS.getMapValueSchema().get(0)));
}
return typeMap;
case STRING:
case INT:
case BIGINT:
case FLOAT:
case DOUBLE:
return pigObj;
case SMALLINT:
if ((Integer) pigObj < Short.MIN_VALUE || (Integer) pigObj > Short.MAX_VALUE) {
handleOutOfRangeValue(pigObj, hcatFS);
return null;
}
return ((Integer) pigObj).shortValue();
case TINYINT:
if ((Integer) pigObj < Byte.MIN_VALUE || (Integer) pigObj > Byte.MAX_VALUE) {
handleOutOfRangeValue(pigObj, hcatFS);
return null;
}
return ((Integer) pigObj).byteValue();
case BOOLEAN:
if (pigObj instanceof String) {
if (((String) pigObj).trim().compareTo("0") == 0) {
return Boolean.FALSE;
}
if (((String) pigObj).trim().compareTo("1") == 0) {
return Boolean.TRUE;
}
throw new BackendException("Unexpected type " + type + " for value " + pigObj + " of class " + pigObj.getClass().getName(), PigHCatUtil.PIG_EXCEPTION_CODE);
}
return Boolean.parseBoolean(pigObj.toString());
case DECIMAL:
BigDecimal bd = (BigDecimal) pigObj;
DecimalTypeInfo dti = (DecimalTypeInfo) hcatFS.getTypeInfo();
if (bd.precision() > dti.precision() || bd.scale() > dti.scale()) {
handleOutOfRangeValue(pigObj, hcatFS);
return null;
}
return HiveDecimal.create(bd);
case CHAR:
String charVal = (String) pigObj;
CharTypeInfo cti = (CharTypeInfo) hcatFS.getTypeInfo();
if (charVal.length() > cti.getLength()) {
handleOutOfRangeValue(pigObj, hcatFS);
return null;
}
return new HiveChar(charVal, cti.getLength());
case VARCHAR:
String varcharVal = (String) pigObj;
VarcharTypeInfo vti = (VarcharTypeInfo) hcatFS.getTypeInfo();
if (varcharVal.length() > vti.getLength()) {
handleOutOfRangeValue(pigObj, hcatFS);
return null;
}
return new HiveVarchar(varcharVal, vti.getLength());
case TIMESTAMP:
DateTime dt = (DateTime) pigObj;
//getMillis() returns UTC time regardless of TZ
return new Timestamp(dt.getMillis());
case DATE:
/**
* We ignore any TZ setting on Pig value since java.sql.Date doesn't have it (in any
* meaningful way). So the assumption is that if Pig value has 0 time component (midnight)
* we assume it reasonably 'fits' into a Hive DATE. If time part is not 0, it's considered
* out of range for target type.
*/
DateTime dateTime = ((DateTime) pigObj);
if (dateTime.getMillisOfDay() != 0) {
handleOutOfRangeValue(pigObj, hcatFS, "Time component must be 0 (midnight) in local timezone; Local TZ val='" + pigObj + "'");
return null;
}
/*java.sql.Date is a poorly defined API. Some (all?) SerDes call toString() on it
[e.g. LazySimpleSerDe, uses LazyUtils.writePrimitiveUTF8()], which automatically adjusts
for local timezone. Date.valueOf() also uses local timezone (as does Date(int,int,int).
Also see PigHCatUtil#extractPigObject() for corresponding read op. This way a DATETIME from Pig,
when stored into Hive and read back comes back with the same value.*/
return new Date(dateTime.getYear() - 1900, dateTime.getMonthOfYear() - 1, dateTime.getDayOfMonth());
default:
throw new BackendException("Unexpected HCat type " + type + " for value " + pigObj + " of class " + pigObj.getClass().getName(), PigHCatUtil.PIG_EXCEPTION_CODE);
}
} catch (BackendException e) {
// provide the path to the field in the error message
throw new BackendException((hcatFS.getName() == null ? " " : hcatFS.getName() + ".") + e.getMessage(), e);
}
}
use of org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo in project presto by prestodb.
the class TestHiveFileFormats method testTruncateVarcharColumn.
@Test
public void testTruncateVarcharColumn() throws Exception {
TestColumn writeColumn = new TestColumn("varchar_column", getPrimitiveJavaObjectInspector(new VarcharTypeInfo(4)), new HiveVarchar("test", 4), utf8Slice("test"));
TestColumn readColumn = new TestColumn("varchar_column", getPrimitiveJavaObjectInspector(new VarcharTypeInfo(3)), new HiveVarchar("tes", 3), utf8Slice("tes"));
assertThatFileFormat(RCTEXT).withWriteColumns(ImmutableList.of(writeColumn)).withReadColumns(ImmutableList.of(readColumn)).isReadableByRecordCursor(new ColumnarTextHiveRecordCursorProvider(HDFS_ENVIRONMENT)).isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
assertThatFileFormat(RCBINARY).withWriteColumns(ImmutableList.of(writeColumn)).withReadColumns(ImmutableList.of(readColumn)).isReadableByRecordCursor(new ColumnarBinaryHiveRecordCursorProvider(HDFS_ENVIRONMENT)).isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
assertThatFileFormat(ORC).withWriteColumns(ImmutableList.of(writeColumn)).withReadColumns(ImmutableList.of(readColumn)).isReadableByPageSource(new OrcPageSourceFactory(TYPE_MANAGER, false, HDFS_ENVIRONMENT));
assertThatFileFormat(PARQUET).withWriteColumns(ImmutableList.of(writeColumn)).withReadColumns(ImmutableList.of(readColumn)).withSession(parquetCursorSession).isReadableByRecordCursor(new ParquetRecordCursorProvider(false, HDFS_ENVIRONMENT));
assertThatFileFormat(PARQUET).withWriteColumns(ImmutableList.of(writeColumn)).withReadColumns(ImmutableList.of(readColumn)).withSession(parquetCursorPushdownSession).isReadableByRecordCursor(new ParquetRecordCursorProvider(false, HDFS_ENVIRONMENT));
assertThatFileFormat(PARQUET).withWriteColumns(ImmutableList.of(writeColumn)).withReadColumns(ImmutableList.of(readColumn)).withSession(parquetPageSourceSession).isReadableByPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, false, HDFS_ENVIRONMENT));
assertThatFileFormat(PARQUET).withWriteColumns(ImmutableList.of(writeColumn)).withReadColumns(ImmutableList.of(readColumn)).withSession(parquetPageSourcePushdown).isReadableByPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, false, HDFS_ENVIRONMENT));
assertThatFileFormat(AVRO).withWriteColumns(ImmutableList.of(writeColumn)).withReadColumns(ImmutableList.of(readColumn)).isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
assertThatFileFormat(SEQUENCEFILE).withWriteColumns(ImmutableList.of(writeColumn)).withReadColumns(ImmutableList.of(readColumn)).isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
assertThatFileFormat(TEXTFILE).withWriteColumns(ImmutableList.of(writeColumn)).withReadColumns(ImmutableList.of(readColumn)).isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
}
use of org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo in project hive by apache.
the class TypeInfoFactory method createPrimitiveTypeInfo.
/**
* Create PrimitiveTypeInfo instance for the given full name of the type. The returned
* type is one of the parameterized type info such as VarcharTypeInfo.
*
* @param fullName Fully qualified name of the type
* @return PrimitiveTypeInfo instance
*/
private static PrimitiveTypeInfo createPrimitiveTypeInfo(String fullName) {
String baseName = TypeInfoUtils.getBaseName(fullName);
PrimitiveTypeEntry typeEntry = PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(baseName);
if (null == typeEntry) {
throw new RuntimeException("Unknown type " + fullName);
}
TypeInfoUtils.PrimitiveParts parts = TypeInfoUtils.parsePrimitiveParts(fullName);
if (parts.typeParams == null || parts.typeParams.length < 1) {
return null;
}
switch(typeEntry.primitiveCategory) {
case CHAR:
if (parts.typeParams.length != 1) {
return null;
}
return new CharTypeInfo(Integer.valueOf(parts.typeParams[0]));
case VARCHAR:
if (parts.typeParams.length != 1) {
return null;
}
return new VarcharTypeInfo(Integer.valueOf(parts.typeParams[0]));
case DECIMAL:
if (parts.typeParams.length != 2) {
return null;
}
return new DecimalTypeInfo(Integer.valueOf(parts.typeParams[0]), Integer.valueOf(parts.typeParams[1]));
default:
return null;
}
}
use of org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo in project hive by apache.
the class SerdeRandomRowSource method randomObject.
public Object randomObject(int column) {
PrimitiveCategory primitiveCategory = primitiveCategories[column];
PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[column];
switch(primitiveCategory) {
case BOOLEAN:
return Boolean.valueOf(r.nextInt(1) == 1);
case BYTE:
return Byte.valueOf((byte) r.nextInt());
case SHORT:
return Short.valueOf((short) r.nextInt());
case INT:
return Integer.valueOf(r.nextInt());
case LONG:
return Long.valueOf(r.nextLong());
case DATE:
return RandomTypeUtil.getRandDate(r);
case FLOAT:
return Float.valueOf(r.nextFloat() * 10 - 5);
case DOUBLE:
return Double.valueOf(r.nextDouble() * 10 - 5);
case STRING:
return RandomTypeUtil.getRandString(r);
case CHAR:
return getRandHiveChar(r, (CharTypeInfo) primitiveTypeInfo);
case VARCHAR:
return getRandHiveVarchar(r, (VarcharTypeInfo) primitiveTypeInfo);
case BINARY:
return getRandBinary(r, 1 + r.nextInt(100));
case TIMESTAMP:
return RandomTypeUtil.getRandTimestamp(r);
case INTERVAL_YEAR_MONTH:
return getRandIntervalYearMonth(r);
case INTERVAL_DAY_TIME:
return getRandIntervalDayTime(r);
case DECIMAL:
{
HiveDecimal dec = getRandHiveDecimal(r, (DecimalTypeInfo) primitiveTypeInfo);
return dec;
}
default:
throw new Error("Unknown primitive category " + primitiveCategory);
}
}
Aggregations