Examples with TimestampWritableV2 org.apache.hadoop.hive.serde2.io.TimestampWritableV2 used on opensource projects

Example 21 with TimestampWritableV2

use of org.apache.hadoop.hive.serde2.io.TimestampWritableV2 in project hive by apache.

the class TestRowContainer method testSpillTimestamp.

@Test
public void testSpillTimestamp() throws HiveException, SerDeException, IOException {
    int blockSize = 10;
    Configuration cfg = new Configuration();
    RowContainer result = new RowContainer(blockSize, cfg, null);
    LazyBinarySerDe serde = new LazyBinarySerDe();
    Properties props = new Properties();
    props.put(serdeConstants.LIST_COLUMNS, "x");
    props.put(serdeConstants.LIST_COLUMN_TYPES, "array<string>");
    serde.initialize(null, props, null);
    result.setSerDe(serde, ObjectInspectorUtils.getStandardObjectInspector(serde.getObjectInspector()));
    result.setTableDesc(PTFRowContainer.createTableDesc((StructObjectInspector) serde.getObjectInspector()));
    TimestampWritableV2 key = new TimestampWritableV2(Timestamp.ofEpochMilli(10));
    result.setKeyObject(Lists.newArrayList(key));
    List<Writable> row;
    // will trigger 2 spills
    for (int i = 0; i <= blockSize * 2; i++) {
        row = new ArrayList<Writable>();
        row.add(new Text("" + i));
        result.addRow(row);
    }
    assertEquals(2, result.getNumFlushedBlocks());
    result.setKeyObject(null);
    assertEquals(Lists.newArrayList(0).toString(), result.first().get(0).toString());
    for (int i = 1; i < result.rowCount() - 1; i++) {
        assertEquals(Lists.newArrayList(i).toString(), result.next().get(0).toString());
    }
    result.close();
}

Example 22 with TimestampWritableV2

use of org.apache.hadoop.hive.serde2.io.TimestampWritableV2 in project hive by apache.

the class TestVectorExpressionWriters method testWriterTimestamp.

private void testWriterTimestamp(TypeInfo type) throws HiveException {
    Timestamp[] timestampValues = new Timestamp[vectorSize];
    TimestampColumnVector tcv = VectorizedRowGroupGenUtil.generateTimestampColumnVector(true, false, vectorSize, new Random(10), timestampValues);
    tcv.isNull[3] = true;
    VectorExpressionWriter vew = getWriter(type);
    for (int i = 0; i < vectorSize; i++) {
        Writable w = (Writable) vew.writeValue(tcv, i);
        if (w != null) {
            Writable expected = getWritableValue(type, timestampValues[i]);
            TimestampWritableV2 t1 = (TimestampWritableV2) expected;
            TimestampWritableV2 t2 = (TimestampWritableV2) w;
            Assert.assertTrue(t1.equals(t2));
        } else {
            Assert.assertTrue(tcv.isNull[i]);
        }
    }
}

Example 23 with TimestampWritableV2

use of org.apache.hadoop.hive.serde2.io.TimestampWritableV2 in project hive by apache.

the class TestVectorTimestampExtract method doRowCastTest.

private boolean doRowCastTest(TypeInfo dateTimeStringTypeInfo, List<String> columns, List<ExprNodeDesc> children, ExprNodeGenericFuncDesc exprDesc, Object[][] randomRows, ObjectInspector rowInspector, Object[] resultObjects) throws Exception {
    /*
    System.out.println(
        "*DEBUG* dateTimeStringTypeInfo " + dateTimeStringTypeInfo.toString() +
        " timestampExtractTestMode ROW_MODE" +
        " exprDesc " + exprDesc.toString());
    */
    HiveConf hiveConf = new HiveConf();
    ExprNodeEvaluator evaluator = ExprNodeEvaluatorFactory.get(exprDesc, hiveConf);
    try {
        evaluator.initialize(rowInspector);
    } catch (HiveException e) {
        return false;
    }
    ObjectInspector objectInspector = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(TypeInfoFactory.intTypeInfo);
    PrimitiveCategory dateTimeStringPrimitiveCategory = ((PrimitiveTypeInfo) dateTimeStringTypeInfo).getPrimitiveCategory();
    final int rowCount = randomRows.length;
    for (int i = 0; i < rowCount; i++) {
        Object[] row = randomRows[i];
        Object object = row[0];
        Object result;
        switch(dateTimeStringPrimitiveCategory) {
            case TIMESTAMP:
                result = evaluator.evaluate((TimestampWritableV2) object);
                break;
            case DATE:
                result = evaluator.evaluate((DateWritableV2) object);
                break;
            case STRING:
                {
                    Text text;
                    if (object == null) {
                        text = null;
                    } else if (object instanceof String) {
                        text = new Text();
                        text.set((String) object);
                    } else {
                        text = (Text) object;
                    }
                    result = evaluator.evaluate(text);
                }
                break;
            default:
                throw new RuntimeException("Unexpected date timestamp string primitive category " + dateTimeStringPrimitiveCategory);
        }
        Object copyResult = ObjectInspectorUtils.copyToStandardObject(result, objectInspector, ObjectInspectorCopyOption.WRITABLE);
        resultObjects[i] = copyResult;
    }
    return true;
}

Example 24 with TimestampWritableV2

use of org.apache.hadoop.hive.serde2.io.TimestampWritableV2 in project hive by apache.

the class TestVectorizedORCReader method checkVectorizedReader.

private void checkVectorizedReader() throws Exception {
    Reader vreader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf));
    Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf));
    RecordReaderImpl vrr = (RecordReaderImpl) vreader.rows();
    RecordReaderImpl rr = (RecordReaderImpl) reader.rows();
    VectorizedRowBatch batch = reader.getSchema().createRowBatchV2();
    OrcStruct row = null;
    long lastRowNumber = -1;
    // Check Vectorized ORC reader against ORC row reader
    while (vrr.nextBatch(batch)) {
        Assert.assertEquals(lastRowNumber + 1, vrr.getRowNumber());
        for (int i = 0; i < batch.size; i++) {
            Assert.assertEquals(rr.getRowNumber(), vrr.getRowNumber() + i);
            lastRowNumber = rr.getRowNumber();
            row = (OrcStruct) rr.next(row);
            for (int j = 0; j < batch.cols.length; j++) {
                Object a = (row.getFieldValue(j));
                ColumnVector cv = batch.cols[j];
                // if the value is repeating, use row 0
                int rowId = cv.isRepeating ? 0 : i;
                // make sure the null flag agrees
                if (a == null) {
                    Assert.assertEquals(true, !cv.noNulls && cv.isNull[rowId]);
                } else if (a instanceof BooleanWritable) {
                    // Boolean values are stores a 1's and 0's, so convert and compare
                    Long temp = (long) (((BooleanWritable) a).get() ? 1 : 0);
                    long b = ((LongColumnVector) cv).vector[rowId];
                    Assert.assertEquals(temp.toString(), Long.toString(b));
                } else if (a instanceof TimestampWritableV2) {
                    // Timestamps are stored as long, so convert and compare
                    TimestampWritableV2 t = ((TimestampWritableV2) a);
                    TimestampColumnVector tcv = ((TimestampColumnVector) cv);
                    java.sql.Timestamp ts = tcv.asScratchTimestamp(rowId);
                    Assert.assertEquals(t.getTimestamp(), Timestamp.ofEpochMilli(ts.getTime(), ts.getNanos()));
                } else if (a instanceof DateWritableV2) {
                    // Dates are stored as long, so convert and compare
                    DateWritableV2 adt = (DateWritableV2) a;
                    long b = ((LongColumnVector) cv).vector[rowId];
                    Assert.assertEquals(adt.get().toEpochMilli(), DateWritableV2.daysToMillis((int) b));
                } else if (a instanceof HiveDecimalWritable) {
                    // Decimals are stored as BigInteger, so convert and compare
                    HiveDecimalWritable dec = (HiveDecimalWritable) a;
                    HiveDecimalWritable b = ((DecimalColumnVector) cv).vector[i];
                    Assert.assertEquals(dec, b);
                } else if (a instanceof DoubleWritable) {
                    double b = ((DoubleColumnVector) cv).vector[rowId];
                    assertEquals(a.toString(), Double.toString(b));
                } else if (a instanceof Text) {
                    BytesColumnVector bcv = (BytesColumnVector) cv;
                    Text b = new Text();
                    b.set(bcv.vector[rowId], bcv.start[rowId], bcv.length[rowId]);
                    assertEquals(a, b);
                } else if (a instanceof IntWritable || a instanceof LongWritable || a instanceof ByteWritable || a instanceof ShortWritable) {
                    assertEquals(a.toString(), Long.toString(((LongColumnVector) cv).vector[rowId]));
                } else {
                    assertEquals("huh", a.getClass().getName());
                }
            }
        }
        // Check repeating
        Assert.assertEquals(false, batch.cols[0].isRepeating);
        Assert.assertEquals(false, batch.cols[1].isRepeating);
        Assert.assertEquals(false, batch.cols[2].isRepeating);
        Assert.assertEquals(true, batch.cols[3].isRepeating);
        Assert.assertEquals(false, batch.cols[4].isRepeating);
        Assert.assertEquals(false, batch.cols[5].isRepeating);
        Assert.assertEquals(false, batch.cols[6].isRepeating);
        Assert.assertEquals(false, batch.cols[7].isRepeating);
        Assert.assertEquals(false, batch.cols[8].isRepeating);
        Assert.assertEquals(false, batch.cols[9].isRepeating);
        // Check non null
        Assert.assertEquals(false, batch.cols[0].noNulls);
        Assert.assertEquals(false, batch.cols[1].noNulls);
        Assert.assertEquals(true, batch.cols[2].noNulls);
        Assert.assertEquals(true, batch.cols[3].noNulls);
        Assert.assertEquals(false, batch.cols[4].noNulls);
        Assert.assertEquals(false, batch.cols[5].noNulls);
        Assert.assertEquals(false, batch.cols[6].noNulls);
        Assert.assertEquals(false, batch.cols[7].noNulls);
        Assert.assertEquals(false, batch.cols[8].noNulls);
        Assert.assertEquals(false, batch.cols[9].noNulls);
    }
    Assert.assertEquals(false, rr.nextBatch(batch));
}

Example 25 with TimestampWritableV2

use of org.apache.hadoop.hive.serde2.io.TimestampWritableV2 in project hive by apache.

the class TestETypeConverter method testGetTimestampConverter.

@Test
public void testGetTimestampConverter() throws Exception {
    Timestamp timestamp = Timestamp.valueOf("2018-06-15 15:12:20.0");
    NanoTime nanoTime = NanoTimeUtils.getNanoTime(timestamp, ZoneOffset.UTC, false);
    PrimitiveType primitiveType = Types.optional(PrimitiveTypeName.INT96).named("value");
    Writable writable = getWritableFromBinaryConverter(null, primitiveType, nanoTime.toBinary());
    TimestampWritableV2 timestampWritable = (TimestampWritableV2) writable;
    assertEquals(timestamp.getNanos(), timestampWritable.getNanos());
}