Examples with HdfsOrcDataSource io.prestosql.plugin.hive.orc.HdfsOrcDataSource used on opensource projects

Example 1 with HdfsOrcDataSource

use of io.prestosql.plugin.hive.orc.HdfsOrcDataSource in project hetu-core by openlookeng.

the class OrcFileWriterFactory method createFileWriter.

@Override
public Optional<HiveFileWriter> createFileWriter(Path path, List<String> inputColumnNames, StorageFormat storageFormat, Properties schema, JobConf configuration, ConnectorSession session, Optional<AcidOutputFormat.Options> acidOptions, Optional<HiveACIDWriteType> acidWriteType) {
    if (!OrcOutputFormat.class.getName().equals(storageFormat.getOutputFormat())) {
        return Optional.empty();
    }
    CompressionKind compression = getCompression(schema, configuration);
    // existing tables and partitions may have columns in a different order than the writer is providing, so build
    // an index to rearrange columns in the proper order
    List<String> fileColumnNames = getColumnNames(schema);
    List<Type> fileColumnTypes = getColumnTypes(schema).stream().map(hiveType -> hiveType.getType(typeManager)).collect(toList());
    List<Type> dataFileColumnTypes = fileColumnTypes;
    int[] fileInputColumnIndexes = fileColumnNames.stream().mapToInt(inputColumnNames::indexOf).toArray();
    Optional<HiveFileWriter> deleteDeltaWriter = Optional.empty();
    if (AcidUtils.isTablePropertyTransactional(schema) && !AcidUtils.isInsertOnlyTable(schema)) {
        ImmutableList<String> orcFileColumnNames = ImmutableList.of(OrcPageSourceFactory.ACID_COLUMN_OPERATION, OrcPageSourceFactory.ACID_COLUMN_ORIGINAL_TRANSACTION, OrcPageSourceFactory.ACID_COLUMN_BUCKET, OrcPageSourceFactory.ACID_COLUMN_ROW_ID, OrcPageSourceFactory.ACID_COLUMN_CURRENT_TRANSACTION, OrcPageSourceFactory.ACID_COLUMN_ROW_STRUCT);
        ImmutableList.Builder<RowType.Field> fieldsBuilder = ImmutableList.builder();
        for (int i = 0; i < fileColumnNames.size(); i++) {
            fieldsBuilder.add(new RowType.Field(Optional.of(fileColumnNames.get(i)), fileColumnTypes.get(i)));
        }
        ImmutableList<Type> orcFileColumnTypes = ImmutableList.of(INTEGER, BIGINT, INTEGER, BIGINT, BIGINT, RowType.from(fieldsBuilder.build()));
        fileColumnNames = orcFileColumnNames;
        fileColumnTypes = orcFileColumnTypes;
        if (acidWriteType.isPresent() && acidWriteType.get() == HiveACIDWriteType.UPDATE) {
            AcidOutputFormat.Options deleteOptions = acidOptions.get().clone().writingDeleteDelta(true);
            Path deleteDeltaPath = AcidUtils.createFilename(path.getParent().getParent(), deleteOptions);
            deleteDeltaWriter = createFileWriter(deleteDeltaPath, inputColumnNames, storageFormat, schema, configuration, session, Optional.of(deleteOptions), Optional.of(HiveACIDWriteType.DELETE));
        }
    }
    try {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(session.getUser(), path, configuration);
        OrcDataSink orcDataSink = createOrcDataSink(session, fileSystem, path);
        Optional<Supplier<OrcDataSource>> validationInputFactory = Optional.empty();
        if (HiveSessionProperties.isOrcOptimizedWriterValidate(session)) {
            validationInputFactory = Optional.of(() -> {
                try {
                    FileStatus fileStatus = fileSystem.getFileStatus(path);
                    return new HdfsOrcDataSource(new OrcDataSourceId(path.toString()), fileStatus.getLen(), HiveSessionProperties.getOrcMaxMergeDistance(session), HiveSessionProperties.getOrcMaxBufferSize(session), HiveSessionProperties.getOrcStreamBufferSize(session), false, fileSystem.open(path), readStats, fileStatus.getModificationTime());
                } catch (IOException e) {
                    throw new PrestoException(HiveErrorCode.HIVE_WRITE_VALIDATION_FAILED, e);
                }
            });
        }
        Callable<Void> rollbackAction = () -> {
            fileSystem.delete(path, false);
            return null;
        };
        return Optional.of(new OrcFileWriter(orcDataSink, rollbackAction, fileColumnNames, fileColumnTypes, dataFileColumnTypes, compression, orcWriterOptions.withStripeMinSize(HiveSessionProperties.getOrcOptimizedWriterMinStripeSize(session)).withStripeMaxSize(HiveSessionProperties.getOrcOptimizedWriterMaxStripeSize(session)).withStripeMaxRowCount(HiveSessionProperties.getOrcOptimizedWriterMaxStripeRows(session)).withDictionaryMaxMemory(HiveSessionProperties.getOrcOptimizedWriterMaxDictionaryMemory(session)).withMaxStringStatisticsLimit(HiveSessionProperties.getOrcStringStatisticsLimit(session)), writeLegacyVersion, fileInputColumnIndexes, ImmutableMap.<String, String>builder().put(HiveMetadata.PRESTO_VERSION_NAME, nodeVersion.toString()).put(HiveMetadata.PRESTO_QUERY_ID_NAME, session.getQueryId()).put("hive.acid.version", String.valueOf(AcidUtils.OrcAcidVersion.ORC_ACID_VERSION)).build(), validationInputFactory, HiveSessionProperties.getOrcOptimizedWriterValidateMode(session), stats, acidOptions, acidWriteType, deleteDeltaWriter, path));
    } catch (IOException e) {
        throw new PrestoException(HiveErrorCode.HIVE_WRITER_OPEN_ERROR, "Error creating ORC file", e);
    }
}

Example 2 with HdfsOrcDataSource

use of io.prestosql.plugin.hive.orc.HdfsOrcDataSource in project hetu-core by openlookeng.

the class HiveWriterFactory method mergeSubFiles.

public void mergeSubFiles(List<HiveWriter> writers) throws IOException {
    if (writers.isEmpty()) {
        return;
    }
    FileSystem fileSystem = hdfsEnvironment.getFileSystem(session.getUser(), new Path(writers.get(0).getFilePath()), conf);
    List<Type> types = dataColumns.stream().map(column -> column.getHiveType().getType(typeManager)).collect(toList());
    for (HiveWriter writer : writers) {
        String filePath = writer.getFilePath();
        Path path = new Path(filePath);
        logContainingFolderInfo(fileSystem, path, "Merging snapshot files to result file: %s", path);
        // The snapshotSuffixes list records the "resumeCount" for each suffix.
        // It doesn't has an entry for the current set of files, so an entry is added first.
        // The resumeCount helps distinguish files created by different runs.
        snapshotSuffixes.add(resumeCount);
        for (int i = 0; i < snapshotSuffixes.size(); i++) {
            long resume = snapshotSuffixes.get(i);
            Path file = new Path(toSnapshotSubFile(filePath, resume, i));
            if (fileSystem.exists(file)) {
                // TODO-cp-I2BZ0A: assuming all files to be of ORC type.
                // Using same parameters as used by SortingFileWriter
                FileStatus fileStatus = fileSystem.getFileStatus(file);
                try (TempFileReader reader = new TempFileReader(types, new HdfsOrcDataSource(new OrcDataSourceId(file.toString()), fileStatus.getLen(), new DataSize(1, MEGABYTE), new DataSize(8, MEGABYTE), new DataSize(8, MEGABYTE), false, fileSystem.open(file), new FileFormatDataSourceStats(), fileStatus.getModificationTime()))) {
                    while (reader.hasNext()) {
                        writer.append(reader.next());
                    }
                }
            // DO NOT delete the sub file, in case we need to resume. Delete them when the query finishes.
            }
        }
    }
}

Example 3 with HdfsOrcDataSource

use of io.prestosql.plugin.hive.orc.HdfsOrcDataSource in project hetu-core by openlookeng.

the class SortingFileWriter method mergeFiles.

private void mergeFiles(Iterable<TempFile> files, Consumer<Page> consumer) {
    try (Closer closer = Closer.create()) {
        Collection<Iterator<Page>> iterators = new ArrayList<>();
        for (TempFile tempFile : files) {
            Path file = tempFile.getPath();
            FileStatus fileStatus = fileSystem.getFileStatus(file);
            OrcDataSource dataSource = new HdfsOrcDataSource(new OrcDataSourceId(file.toString()), fileStatus.getLen(), new DataSize(1, MEGABYTE), new DataSize(8, MEGABYTE), new DataSize(8, MEGABYTE), false, fileSystem.open(file), new FileFormatDataSourceStats(), fileStatus.getModificationTime());
            TempFileReader reader = new TempFileReader(types, dataSource);
            // Closing the reader also closes the data source
            closer.register(reader);
            iterators.add(reader);
        }
        new MergingPageIterator(iterators, types, sortFields, sortOrders).forEachRemaining(consumer);
        for (TempFile tempFile : files) {
            Path file = tempFile.getPath();
            fileSystem.delete(file, false);
            if (fileSystem.exists(file)) {
                throw new IOException("Failed to delete temporary file: " + file);
            }
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}