Examples with ExtendedRecordWriter com.facebook.presto.hive.RecordFileWriter.ExtendedRecordWriter used on opensource projects

Example 1 with ExtendedRecordWriter

use of com.facebook.presto.hive.RecordFileWriter.ExtendedRecordWriter in project presto by prestodb.

the class ParquetRecordWriterUtil method createParquetWriter.

public static RecordWriter createParquetWriter(Path target, JobConf conf, Properties properties, boolean compress, ConnectorSession session) throws IOException, ReflectiveOperationException {
    conf.setLong(ParquetOutputFormat.BLOCK_SIZE, getParquetWriterBlockSize(session).toBytes());
    conf.setLong(ParquetOutputFormat.PAGE_SIZE, getParquetWriterPageSize(session).toBytes());
    RecordWriter recordWriter = new MapredParquetOutputFormat().getHiveRecordWriter(conf, target, Text.class, compress, properties, Reporter.NULL);
    Object realWriter = REAL_WRITER_FIELD.get(recordWriter);
    Object internalWriter = INTERNAL_WRITER_FIELD.get(realWriter);
    ParquetFileWriter fileWriter = (ParquetFileWriter) FILE_WRITER_FIELD.get(internalWriter);
    return new ExtendedRecordWriter() {

        private long length;

        @Override
        public long getWrittenBytes() {
            return length;
        }

        @Override
        public void write(Writable value) throws IOException {
            recordWriter.write(value);
            length = fileWriter.getPos();
        }

        @Override
        public void close(boolean abort) throws IOException {
            recordWriter.close(abort);
            if (!abort) {
                length = target.getFileSystem(conf).getFileStatus(target).getLen();
            }
        }
    };
}

Example 2 with ExtendedRecordWriter

use of com.facebook.presto.hive.RecordFileWriter.ExtendedRecordWriter in project presto by prestodb.

the class HiveWriteUtils method createRcFileWriter.

private static RecordWriter createRcFileWriter(Path target, JobConf conf, Properties properties, boolean compress) throws IOException {
    int columns = properties.getProperty(META_TABLE_COLUMNS).split(",").length;
    RCFileOutputFormat.setColumnNumber(conf, columns);
    CompressionCodec codec = null;
    if (compress) {
        codec = ReflectionUtil.newInstance(getOutputCompressorClass(conf, DefaultCodec.class), conf);
    }
    RCFile.Writer writer = new RCFile.Writer(target.getFileSystem(conf), conf, target, () -> {
    }, codec);
    return new ExtendedRecordWriter() {

        private long length;

        @Override
        public long getWrittenBytes() {
            return length;
        }

        @Override
        public void write(Writable value) throws IOException {
            writer.append(value);
            length = writer.getLength();
        }

        @Override
        public void close(boolean abort) throws IOException {
            writer.close();
            if (!abort) {
                length = target.getFileSystem(conf).getFileStatus(target).getLen();
            }
        }
    };
}