Examples with ThreadLocalClassLoaderConfiguration org.apache.flink.orc.writer.ThreadLocalClassLoaderConfiguration used on opensource projects

Example 1 with ThreadLocalClassLoaderConfiguration

use of org.apache.flink.orc.writer.ThreadLocalClassLoaderConfiguration in project flink by apache.

the class HiveTableSink method createBulkWriterFactory.

private Optional<BulkWriter.Factory<RowData>> createBulkWriterFactory(String[] partitionColumns, StorageDescriptor sd) {
    String serLib = sd.getSerdeInfo().getSerializationLib().toLowerCase();
    int formatFieldCount = tableSchema.getFieldCount() - partitionColumns.length;
    String[] formatNames = new String[formatFieldCount];
    LogicalType[] formatTypes = new LogicalType[formatFieldCount];
    for (int i = 0; i < formatFieldCount; i++) {
        formatNames[i] = tableSchema.getFieldName(i).get();
        formatTypes[i] = tableSchema.getFieldDataType(i).get().getLogicalType();
    }
    RowType formatType = RowType.of(formatTypes, formatNames);
    if (serLib.contains("parquet")) {
        Configuration formatConf = new Configuration(jobConf);
        sd.getSerdeInfo().getParameters().forEach(formatConf::set);
        return Optional.of(ParquetRowDataBuilder.createWriterFactory(formatType, formatConf, hiveVersion.startsWith("3.")));
    } else if (serLib.contains("orc")) {
        Configuration formatConf = new ThreadLocalClassLoaderConfiguration(jobConf);
        sd.getSerdeInfo().getParameters().forEach(formatConf::set);
        TypeDescription typeDescription = OrcSplitReaderUtil.logicalTypeToOrcType(formatType);
        return Optional.of(hiveShim.createOrcBulkWriterFactory(formatConf, typeDescription.toString(), formatTypes));
    } else {
        return Optional.empty();
    }
}