Search in sources :

Example 11 with OrcSerde

use of org.apache.hadoop.hive.ql.io.orc.OrcSerde in project presto by prestodb.

the class TestOrcReaderPositions method createSequentialFile.

private static void createSequentialFile(File file, int count) throws IOException, SerDeException {
    FileSinkOperator.RecordWriter writer = createOrcRecordWriter(file, ORC_12, CompressionKind.NONE, BIGINT);
    Serializer serde = new OrcSerde();
    SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", BIGINT);
    Object row = objectInspector.create();
    StructField field = objectInspector.getAllStructFieldRefs().get(0);
    for (int i = 0; i < count; i++) {
        objectInspector.setStructFieldData(row, field, (long) i);
        Writable record = serde.serialize(row, objectInspector);
        writer.write(record);
    }
    writer.close(false);
}
Also used : SettableStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector) OrcTester.createSettableStructObjectInspector(com.facebook.presto.orc.OrcTester.createSettableStructObjectInspector) StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) FileSinkOperator(org.apache.hadoop.hive.ql.exec.FileSinkOperator) OrcSerde(org.apache.hadoop.hive.ql.io.orc.OrcSerde) Writable(org.apache.hadoop.io.Writable) Serializer(org.apache.hadoop.hive.serde2.Serializer)

Example 12 with OrcSerde

use of org.apache.hadoop.hive.ql.io.orc.OrcSerde in project presto by prestodb.

the class TestOrcReaderPositions method createMultiStripeFile.

// write 5 stripes of 20 values each: (0,3,6,..,57), (60,..,117), .., (..297)
private static void createMultiStripeFile(File file) throws IOException, ReflectiveOperationException, SerDeException {
    FileSinkOperator.RecordWriter writer = createOrcRecordWriter(file, ORC_12, CompressionKind.NONE, ImmutableList.of(BIGINT, VARCHAR));
    Serializer serde = new OrcSerde();
    SettableStructObjectInspector objectInspector = createSettableStructObjectInspector(ImmutableList.of(BIGINT, VARCHAR));
    Object row = objectInspector.create();
    StructField bigintField = objectInspector.getAllStructFieldRefs().get(0);
    StructField varcharField = objectInspector.getAllStructFieldRefs().get(1);
    for (int i = 0; i < 300; i += 3) {
        if ((i > 0) && (i % 60 == 0)) {
            flushWriter(writer);
        }
        objectInspector.setStructFieldData(row, bigintField, (long) i);
        objectInspector.setStructFieldData(row, varcharField, String.valueOf(i));
        Writable record = serde.serialize(row, objectInspector);
        writer.write(record);
    }
    writer.close(false);
}
Also used : SettableStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector) OrcTester.createSettableStructObjectInspector(com.facebook.presto.orc.OrcTester.createSettableStructObjectInspector) StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) FileSinkOperator(org.apache.hadoop.hive.ql.exec.FileSinkOperator) OrcSerde(org.apache.hadoop.hive.ql.io.orc.OrcSerde) Writable(org.apache.hadoop.io.Writable) Serializer(org.apache.hadoop.hive.serde2.Serializer)

Example 13 with OrcSerde

use of org.apache.hadoop.hive.ql.io.orc.OrcSerde in project presto by prestodb.

the class TestOrcMapNullKey method createSingleColumnMapFileWithNullValues.

private static TempFile createSingleColumnMapFileWithNullValues(Type mapType, Map<Long, Long> map) throws IOException {
    OrcSerde serde = new OrcSerde();
    TempFile tempFile = new TempFile();
    FileSinkOperator.RecordWriter writer = createOrcRecordWriter(tempFile.getFile(), ORC_12, CompressionKind.NONE, mapType);
    SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", mapType);
    Object row = objectInspector.create();
    StructField field = objectInspector.getAllStructFieldRefs().get(0);
    objectInspector.setStructFieldData(row, field, map);
    Writable record = serde.serialize(row, objectInspector);
    writer.write(record);
    writer.close(false);
    return tempFile;
}
Also used : SettableStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector) OrcTester.createSettableStructObjectInspector(com.facebook.presto.orc.OrcTester.createSettableStructObjectInspector) StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) FileSinkOperator(org.apache.hadoop.hive.ql.exec.FileSinkOperator) OrcSerde(org.apache.hadoop.hive.ql.io.orc.OrcSerde) Writable(org.apache.hadoop.io.Writable)

Aggregations

OrcSerde (org.apache.hadoop.hive.ql.io.orc.OrcSerde)13 SettableStructObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector)9 StructField (org.apache.hadoop.hive.serde2.objectinspector.StructField)9 Writable (org.apache.hadoop.io.Writable)9 OrcTester.createSettableStructObjectInspector (com.facebook.presto.orc.OrcTester.createSettableStructObjectInspector)8 Serializer (org.apache.hadoop.hive.serde2.Serializer)8 FileSinkOperator (org.apache.hadoop.hive.ql.exec.FileSinkOperator)7 DataXException (com.alibaba.datax.common.exception.DataXException)2 IOException (java.io.IOException)2 StructObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector)2 Record (com.alibaba.datax.common.element.Record)1 Configuration (com.alibaba.datax.common.util.Configuration)1 ColumnEntry (com.alibaba.datax.plugin.unstructuredstorage.reader.ColumnEntry)1 JSONObject (com.alibaba.fastjson.JSONObject)1 OrcTester.createOrcRecordWriter (com.facebook.presto.orc.OrcTester.createOrcRecordWriter)1 HashMap (java.util.HashMap)1 Path (org.apache.hadoop.fs.Path)1 RecordWriter (org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter)1 RCFileRecordReader (org.apache.hadoop.hive.ql.io.RCFileRecordReader)1 OrcInputFormat (org.apache.hadoop.hive.ql.io.orc.OrcInputFormat)1