Examples with TimestampWritableV2 org.apache.hadoop.hive.serde2.io.TimestampWritableV2 used on opensource projects

Example 81 with TimestampWritableV2

use of org.apache.hadoop.hive.serde2.io.TimestampWritableV2 in project hive by apache.

the class TestGenericUDFOPPlus method testDatePlusIntervalDayTime.

@Test
public void testDatePlusIntervalDayTime() throws Exception {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();
    DateWritableV2 left = new DateWritableV2(Date.valueOf("2001-01-01"));
    HiveIntervalDayTimeWritable right = new HiveIntervalDayTimeWritable(HiveIntervalDayTime.valueOf("1 2:3:4.567"));
    ObjectInspector[] inputOIs = { PrimitiveObjectInspectorFactory.writableDateObjectInspector, PrimitiveObjectInspectorFactory.writableHiveIntervalDayTimeObjectInspector };
    DeferredObject[] args = { new DeferredJavaObject(left), new DeferredJavaObject(right) };
    // Date + day-time interval = timestamp
    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.timestampTypeInfo, oi.getTypeInfo());
    TimestampWritableV2 res = (TimestampWritableV2) udf.evaluate(args);
    Assert.assertEquals(Timestamp.valueOf("2001-01-02 2:3:4.567"), res.getTimestamp());
}

Example 82 with TimestampWritableV2

use of org.apache.hadoop.hive.serde2.io.TimestampWritableV2 in project hive by apache.

the class TestGenericUDFOPPlus method testIntervalDayTimePlusTimestamp.

@Test
public void testIntervalDayTimePlusTimestamp() throws Exception {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();
    HiveIntervalDayTimeWritable left = new HiveIntervalDayTimeWritable(HiveIntervalDayTime.valueOf("1 2:3:4.567"));
    TimestampWritableV2 right = new TimestampWritableV2(Timestamp.valueOf("2001-01-01 00:00:00"));
    ObjectInspector[] inputOIs = { PrimitiveObjectInspectorFactory.writableHiveIntervalDayTimeObjectInspector, PrimitiveObjectInspectorFactory.writableTimestampObjectInspector };
    DeferredObject[] args = { new DeferredJavaObject(left), new DeferredJavaObject(right) };
    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.timestampTypeInfo, oi.getTypeInfo());
    TimestampWritableV2 res = (TimestampWritableV2) udf.evaluate(args);
    Assert.assertEquals(Timestamp.valueOf("2001-01-02 2:3:4.567"), res.getTimestamp());
}

Example 83 with TimestampWritableV2

use of org.apache.hadoop.hive.serde2.io.TimestampWritableV2 in project hive by apache.

the class TestOrcFile method testUnionAndTimestamp.

/**
 * We test union, timestamp, and decimal separately since we need to make the
 * object inspector manually. (The Hive reflection-based doesn't handle
 * them properly.)
 */
@Test
public void testUnionAndTimestamp() throws Exception {
    List<OrcProto.Type> types = new ArrayList<OrcProto.Type>();
    types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.STRUCT).addFieldNames("time").addFieldNames("union").addFieldNames("decimal").addSubtypes(1).addSubtypes(2).addSubtypes(5).build());
    types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.TIMESTAMP).build());
    types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.UNION).addSubtypes(3).addSubtypes(4).build());
    types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.INT).build());
    types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.STRING).build());
    types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.DECIMAL).build());
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
        inspector = OrcStruct.createObjectInspector(0, types);
    }
    HiveDecimal maxValue = HiveDecimal.create("10000000000000000000");
    Writer writer = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf).inspector(inspector).stripeSize(1000).compress(CompressionKind.NONE).batchSize(1000).bufferSize(100).blockPadding(false));
    OrcStruct row = new OrcStruct(3);
    OrcUnion union = new OrcUnion();
    row.setFieldValue(1, union);
    row.setFieldValue(0, new TimestampWritableV2(Timestamp.valueOf("2000-03-12 15:00:00")));
    HiveDecimal value = HiveDecimal.create("12345678.6547456");
    row.setFieldValue(2, new HiveDecimalWritable(value));
    union.set((byte) 0, new IntWritable(42));
    writer.addRow(row);
    row.setFieldValue(0, new TimestampWritableV2(Timestamp.valueOf("2000-03-20 12:00:00.123456789")));
    union.set((byte) 1, new Text("hello"));
    value = HiveDecimal.create("-5643.234");
    row.setFieldValue(2, new HiveDecimalWritable(value));
    writer.addRow(row);
    row.setFieldValue(0, null);
    row.setFieldValue(1, null);
    row.setFieldValue(2, null);
    writer.addRow(row);
    row.setFieldValue(1, union);
    union.set((byte) 0, null);
    writer.addRow(row);
    union.set((byte) 1, null);
    writer.addRow(row);
    union.set((byte) 0, new IntWritable(200000));
    row.setFieldValue(0, new TimestampWritableV2(Timestamp.valueOf("1970-01-01 00:00:00")));
    value = HiveDecimal.create("10000000000000000000");
    row.setFieldValue(2, new HiveDecimalWritable(value));
    writer.addRow(row);
    Random rand = new Random(42);
    for (int i = 1970; i < 2038; ++i) {
        row.setFieldValue(0, new TimestampWritableV2(Timestamp.valueOf(i + "-05-05 12:34:56." + i)));
        if ((i & 1) == 0) {
            union.set((byte) 0, new IntWritable(i * i));
        } else {
            union.set((byte) 1, new Text(Integer.toString(i * i)));
        }
        value = HiveDecimal.create(new BigInteger(64, rand), rand.nextInt(18));
        row.setFieldValue(2, new HiveDecimalWritable(value));
        if (maxValue.compareTo(value) < 0) {
            maxValue = value;
        }
        writer.addRow(row);
    }
    // let's add a lot of constant rows to test the rle
    row.setFieldValue(0, null);
    union.set((byte) 0, new IntWritable(1732050807));
    row.setFieldValue(2, null);
    for (int i = 0; i < 5000; ++i) {
        writer.addRow(row);
    }
    union.set((byte) 0, new IntWritable(0));
    writer.addRow(row);
    union.set((byte) 0, new IntWritable(10));
    writer.addRow(row);
    union.set((byte) 0, new IntWritable(138));
    writer.addRow(row);
    writer.close();
    Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs));
    TypeDescription schema = writer.getSchema();
    assertEquals(5, schema.getMaximumId());
    boolean[] expected = new boolean[] { false, false, false, false, false, false };
    boolean[] included = OrcUtils.includeColumns("", schema);
    assertEquals(true, Arrays.equals(expected, included));
    expected = new boolean[] { false, true, false, false, false, true };
    included = OrcUtils.includeColumns("time,decimal", schema);
    assertEquals(true, Arrays.equals(expected, included));
    expected = new boolean[] { false, false, true, true, true, false };
    included = OrcUtils.includeColumns("union", schema);
    assertEquals(true, Arrays.equals(expected, included));
    assertEquals(false, reader.getMetadataKeys().iterator().hasNext());
    assertEquals(5077, reader.getNumberOfRows());
    DecimalColumnStatistics stats = (DecimalColumnStatistics) reader.getStatistics()[5];
    assertEquals(71, stats.getNumberOfValues());
    assertEquals(HiveDecimal.create("-5643.234"), stats.getMinimum());
    assertEquals(maxValue, stats.getMaximum());
    // TODO: fix this
    // assertEquals(null,stats.getSum());
    int stripeCount = 0;
    int rowCount = 0;
    long currentOffset = -1;
    for (StripeInformation stripe : reader.getStripes()) {
        stripeCount += 1;
        rowCount += stripe.getNumberOfRows();
        if (currentOffset < 0) {
            currentOffset = stripe.getOffset() + stripe.getLength();
        } else {
            assertEquals(currentOffset, stripe.getOffset());
            currentOffset += stripe.getLength();
        }
    }
    assertEquals(reader.getNumberOfRows(), rowCount);
    assertEquals(2, stripeCount);
    assertEquals(reader.getContentLength(), currentOffset);
    RecordReader rows = reader.rows();
    assertEquals(0, rows.getRowNumber());
    assertEquals(0.0, rows.getProgress(), 0.000001);
    assertEquals(true, rows.hasNext());
    row = (OrcStruct) rows.next(null);
    assertEquals(1, rows.getRowNumber());
    inspector = reader.getObjectInspector();
    assertEquals("struct<time:timestamp,union:uniontype<int,string>,decimal:decimal(38,18)>", inspector.getTypeName());
    assertEquals(new TimestampWritableV2(Timestamp.valueOf("2000-03-12 15:00:00")), row.getFieldValue(0));
    union = (OrcUnion) row.getFieldValue(1);
    assertEquals(0, union.getTag());
    assertEquals(new IntWritable(42), union.getObject());
    assertEquals(new HiveDecimalWritable(HiveDecimal.create("12345678.6547456")), row.getFieldValue(2));
    row = (OrcStruct) rows.next(row);
    assertEquals(2, rows.getRowNumber());
    assertEquals(new TimestampWritableV2(Timestamp.valueOf("2000-03-20 12:00:00.123456789")), row.getFieldValue(0));
    assertEquals(1, union.getTag());
    assertEquals(new Text("hello"), union.getObject());
    assertEquals(new HiveDecimalWritable(HiveDecimal.create("-5643.234")), row.getFieldValue(2));
    row = (OrcStruct) rows.next(row);
    assertEquals(null, row.getFieldValue(0));
    assertEquals(null, row.getFieldValue(1));
    assertEquals(null, row.getFieldValue(2));
    row = (OrcStruct) rows.next(row);
    assertEquals(null, row.getFieldValue(0));
    union = (OrcUnion) row.getFieldValue(1);
    assertEquals(0, union.getTag());
    assertEquals(null, union.getObject());
    assertEquals(null, row.getFieldValue(2));
    row = (OrcStruct) rows.next(row);
    assertEquals(null, row.getFieldValue(0));
    assertEquals(1, union.getTag());
    assertEquals(null, union.getObject());
    assertEquals(null, row.getFieldValue(2));
    row = (OrcStruct) rows.next(row);
    assertEquals(new TimestampWritableV2(Timestamp.valueOf("1970-01-01 00:00:00")), row.getFieldValue(0));
    assertEquals(new IntWritable(200000), union.getObject());
    assertEquals(new HiveDecimalWritable(HiveDecimal.create("10000000000000000000")), row.getFieldValue(2));
    rand = new Random(42);
    for (int i = 1970; i < 2038; ++i) {
        row = (OrcStruct) rows.next(row);
        assertEquals(new TimestampWritableV2(Timestamp.valueOf(i + "-05-05 12:34:56." + i)), row.getFieldValue(0));
        if ((i & 1) == 0) {
            assertEquals(0, union.getTag());
            assertEquals(new IntWritable(i * i), union.getObject());
        } else {
            assertEquals(1, union.getTag());
            assertEquals(new Text(Integer.toString(i * i)), union.getObject());
        }
        assertEquals(new HiveDecimalWritable(HiveDecimal.create(new BigInteger(64, rand), rand.nextInt(18))), row.getFieldValue(2));
    }
    for (int i = 0; i < 5000; ++i) {
        row = (OrcStruct) rows.next(row);
        assertEquals(new IntWritable(1732050807), union.getObject());
    }
    row = (OrcStruct) rows.next(row);
    assertEquals(new IntWritable(0), union.getObject());
    row = (OrcStruct) rows.next(row);
    assertEquals(new IntWritable(10), union.getObject());
    row = (OrcStruct) rows.next(row);
    assertEquals(new IntWritable(138), union.getObject());
    assertEquals(false, rows.hasNext());
    assertEquals(1.0, rows.getProgress(), 0.00001);
    assertEquals(reader.getNumberOfRows(), rows.getRowNumber());
    rows.seekToRow(1);
    row = (OrcStruct) rows.next(row);
    assertEquals(new TimestampWritableV2(Timestamp.valueOf("2000-03-20 12:00:00.123456789")), row.getFieldValue(0));
    assertEquals(1, union.getTag());
    assertEquals(new Text("hello"), union.getObject());
    assertEquals(new HiveDecimalWritable(HiveDecimal.create("-5643.234")), row.getFieldValue(2));
    rows.close();
}

Example 84 with TimestampWritableV2

use of org.apache.hadoop.hive.serde2.io.TimestampWritableV2 in project hive by apache.

the class TestOrcFile method createOrcDateFile.

/**
 * Generate an ORC file with a range of dates and times.
 */
public void createOrcDateFile(Path file, int minYear, int maxYear) throws IOException {
    List<OrcProto.Type> types = new ArrayList<OrcProto.Type>();
    types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.STRUCT).addFieldNames("time").addFieldNames("date").addSubtypes(1).addSubtypes(2).build());
    types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.TIMESTAMP).build());
    types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.DATE).build());
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
        inspector = OrcStruct.createObjectInspector(0, types);
    }
    Writer writer = OrcFile.createWriter(file, OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).bufferSize(10000).blockPadding(false).setProlepticGregorian(true));
    OrcStruct row = new OrcStruct(2);
    for (int year = minYear; year < maxYear; ++year) {
        for (int ms = 1000; ms < 2000; ++ms) {
            row.setFieldValue(0, new TimestampWritableV2(Timestamp.valueOf(year + "-05-05 12:34:56." + ms)));
            row.setFieldValue(1, new DateWritableV2(Date.of(year - 1900, 11, 25)));
            writer.addRow(row);
        }
    }
    writer.close();
    Reader reader = OrcFile.createReader(file, OrcFile.readerOptions(conf));
    RecordReader rows = reader.rows();
    for (int year = minYear; year < maxYear; ++year) {
        for (int ms = 1000; ms < 2000; ++ms) {
            row = (OrcStruct) rows.next(row);
            assertEquals(new TimestampWritableV2(Timestamp.valueOf(year + "-05-05 12:34:56." + ms)), row.getFieldValue(0));
            assertEquals(new DateWritableV2(Date.of(year - 1900, 11, 25)), row.getFieldValue(1));
        }
    }
    rows.close();
}

Example 85 with TimestampWritableV2

use of org.apache.hadoop.hive.serde2.io.TimestampWritableV2 in project hive by apache.

the class TestVectorExpressionWriters method testSetterTimestamp.

private void testSetterTimestamp(TypeInfo type) throws HiveException {
    Timestamp[] timestampValues = new Timestamp[vectorSize];
    TimestampColumnVector tcv = VectorizedRowGroupGenUtil.generateTimestampColumnVector(true, false, vectorSize, new Random(10), timestampValues);
    tcv.isNull[3] = true;
    Object[] values = new Object[this.vectorSize];
    VectorExpressionWriter vew = getWriter(type);
    for (int i = 0; i < vectorSize; i++) {
        // setValue() should be able to handle null input
        values[i] = null;
        values[i] = vew.setValue(values[i], tcv, i);
        if (values[i] != null) {
            Writable expected = getWritableValue(type, timestampValues[i]);
            TimestampWritableV2 t1 = (TimestampWritableV2) expected;
            TimestampWritableV2 t2 = (TimestampWritableV2) values[i];
            Assert.assertTrue(t1.equals(t2));
        } else {
            Assert.assertTrue(tcv.isNull[i]);
        }
    }
}