Search in sources :

Example 71 with PrimitiveType

use of org.apache.parquet.schema.PrimitiveType in project hive by apache.

the class TestETypeConverter method testGetInt64NanosTimestampConverter.

@Test
public void testGetInt64NanosTimestampConverter() throws Exception {
    Timestamp timestamp = Timestamp.valueOf("2018-07-15 15:12:20.11223344");
    PrimitiveType primitiveType = createInt64TimestampType(false, TimeUnit.NANOS);
    long time = timestamp.toEpochSecond() * 1000000000 + timestamp.getNanos();
    Writable writable = getWritableFromPrimitiveConverter(null, primitiveType, time);
    TimestampWritableV2 timestampWritable = (TimestampWritableV2) writable;
    assertEquals(timestamp.toEpochMilli(), timestampWritable.getTimestamp().toEpochMilli());
    assertEquals(timestamp.getNanos(), timestampWritable.getNanos());
}
Also used : Writable(org.apache.hadoop.io.Writable) DoubleWritable(org.apache.hadoop.io.DoubleWritable) LongWritable(org.apache.hadoop.io.LongWritable) BytesWritable(org.apache.hadoop.io.BytesWritable) IntWritable(org.apache.hadoop.io.IntWritable) BooleanWritable(org.apache.hadoop.io.BooleanWritable) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) FloatWritable(org.apache.hadoop.io.FloatWritable) PrimitiveType(org.apache.parquet.schema.PrimitiveType) Timestamp(org.apache.hadoop.hive.common.type.Timestamp) TimestampWritableV2(org.apache.hadoop.hive.serde2.io.TimestampWritableV2) Test(org.junit.Test)

Example 72 with PrimitiveType

use of org.apache.parquet.schema.PrimitiveType in project hive by apache.

the class TestETypeConverter method testGetIntConverterNoHiveTypeInfo.

@Test
public void testGetIntConverterNoHiveTypeInfo() throws Exception {
    PrimitiveType primitiveType = Types.optional(PrimitiveTypeName.INT32).named("value");
    Writable writable = getWritableFromPrimitiveConverter(null, primitiveType, 12225);
    IntWritable intWritable = (IntWritable) writable;
    assertEquals(12225, intWritable.get());
}
Also used : Writable(org.apache.hadoop.io.Writable) DoubleWritable(org.apache.hadoop.io.DoubleWritable) LongWritable(org.apache.hadoop.io.LongWritable) BytesWritable(org.apache.hadoop.io.BytesWritable) IntWritable(org.apache.hadoop.io.IntWritable) BooleanWritable(org.apache.hadoop.io.BooleanWritable) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) FloatWritable(org.apache.hadoop.io.FloatWritable) PrimitiveType(org.apache.parquet.schema.PrimitiveType) IntWritable(org.apache.hadoop.io.IntWritable) Test(org.junit.Test)

Example 73 with PrimitiveType

use of org.apache.parquet.schema.PrimitiveType in project hive by apache.

the class TestETypeConverter method testGetTextConverterNoHiveTypeInfo.

@Test
public void testGetTextConverterNoHiveTypeInfo() throws Exception {
    PrimitiveType primitiveType = Types.optional(PrimitiveTypeName.BINARY).as(LogicalTypeAnnotation.stringType()).named("value");
    Writable writable = getWritableFromBinaryConverter(null, primitiveType, Binary.fromString("this_is_a_value"));
    Text textWritable = (Text) writable;
    assertEquals("this_is_a_value", textWritable.toString());
}
Also used : Writable(org.apache.hadoop.io.Writable) DoubleWritable(org.apache.hadoop.io.DoubleWritable) LongWritable(org.apache.hadoop.io.LongWritable) BytesWritable(org.apache.hadoop.io.BytesWritable) IntWritable(org.apache.hadoop.io.IntWritable) BooleanWritable(org.apache.hadoop.io.BooleanWritable) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) FloatWritable(org.apache.hadoop.io.FloatWritable) PrimitiveType(org.apache.parquet.schema.PrimitiveType) Text(org.apache.hadoop.io.Text) Test(org.junit.Test)

Example 74 with PrimitiveType

use of org.apache.parquet.schema.PrimitiveType in project hive by apache.

the class TestETypeConverter method testGetBinaryConverter.

@Test
public void testGetBinaryConverter() throws Exception {
    PrimitiveType primitiveType = Types.optional(PrimitiveTypeName.BINARY).named("value");
    Writable writable = getWritableFromBinaryConverter(null, primitiveType, Binary.fromString("this_is_a_value"));
    BytesWritable byteWritable = (BytesWritable) writable;
    assertEquals("this_is_a_value", new String(byteWritable.getBytes()));
}
Also used : Writable(org.apache.hadoop.io.Writable) DoubleWritable(org.apache.hadoop.io.DoubleWritable) LongWritable(org.apache.hadoop.io.LongWritable) BytesWritable(org.apache.hadoop.io.BytesWritable) IntWritable(org.apache.hadoop.io.IntWritable) BooleanWritable(org.apache.hadoop.io.BooleanWritable) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) FloatWritable(org.apache.hadoop.io.FloatWritable) PrimitiveType(org.apache.parquet.schema.PrimitiveType) BytesWritable(org.apache.hadoop.io.BytesWritable) Test(org.junit.Test)

Example 75 with PrimitiveType

use of org.apache.parquet.schema.PrimitiveType in project hive by apache.

the class TestETypeConverter method testGetDoubleConverter.

@Test
public void testGetDoubleConverter() throws Exception {
    MyConverterParent converterParent = new MyConverterParent();
    PrimitiveType primitiveType = Types.optional(PrimitiveTypeName.DOUBLE).named("value");
    PrimitiveConverter converter = ETypeConverter.getNewConverter(primitiveType, 1, converterParent, null);
    ((PrimitiveConverter) converter).addDouble(3276);
    Writable writable = converterParent.getValue();
    DoubleWritable doubleWritable = (DoubleWritable) writable;
    assertEquals(3276, doubleWritable.get(), 0);
}
Also used : PrimitiveConverter(org.apache.parquet.io.api.PrimitiveConverter) Writable(org.apache.hadoop.io.Writable) DoubleWritable(org.apache.hadoop.io.DoubleWritable) LongWritable(org.apache.hadoop.io.LongWritable) BytesWritable(org.apache.hadoop.io.BytesWritable) IntWritable(org.apache.hadoop.io.IntWritable) BooleanWritable(org.apache.hadoop.io.BooleanWritable) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) FloatWritable(org.apache.hadoop.io.FloatWritable) PrimitiveType(org.apache.parquet.schema.PrimitiveType) DoubleWritable(org.apache.hadoop.io.DoubleWritable) Test(org.junit.Test)

Aggregations

PrimitiveType (org.apache.parquet.schema.PrimitiveType)123 Test (org.junit.Test)66 MessageType (org.apache.parquet.schema.MessageType)41 HiveDecimalWritable (org.apache.hadoop.hive.serde2.io.HiveDecimalWritable)28 BooleanWritable (org.apache.hadoop.io.BooleanWritable)28 BytesWritable (org.apache.hadoop.io.BytesWritable)28 DoubleWritable (org.apache.hadoop.io.DoubleWritable)28 FloatWritable (org.apache.hadoop.io.FloatWritable)28 IntWritable (org.apache.hadoop.io.IntWritable)28 LongWritable (org.apache.hadoop.io.LongWritable)28 Writable (org.apache.hadoop.io.Writable)28 GroupType (org.apache.parquet.schema.GroupType)25 Test (org.testng.annotations.Test)20 ColumnDescriptor (org.apache.parquet.column.ColumnDescriptor)14 OriginalType (org.apache.parquet.schema.OriginalType)14 Type (org.apache.parquet.schema.Type)12 List (java.util.List)11 ColumnIndex (org.apache.parquet.internal.column.columnindex.ColumnIndex)11 ColumnIndexBuilder (org.apache.parquet.internal.column.columnindex.ColumnIndexBuilder)11 ArrayList (java.util.ArrayList)10