Examples with HdfsContext com.facebook.presto.hive.HdfsContext used on opensource projects

Example 6 with HdfsContext

use of com.facebook.presto.hive.HdfsContext in project presto by prestodb.

the class IcebergHiveMetadata method createSchema.

@Override
public void createSchema(ConnectorSession session, String schemaName, Map<String, Object> properties) {
    Optional<String> location = getSchemaLocation(properties).map(uri -> {
        try {
            hdfsEnvironment.getFileSystem(new HdfsContext(session, schemaName), new Path(uri));
        } catch (IOException | IllegalArgumentException e) {
            throw new PrestoException(INVALID_SCHEMA_PROPERTY, "Invalid location URI: " + uri, e);
        }
        return uri;
    });
    Database database = Database.builder().setDatabaseName(schemaName).setLocation(location).setOwnerType(USER).setOwnerName(session.getUser()).build();
    MetastoreContext metastoreContext = new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), Optional.empty(), false, HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER);
    metastore.createDatabase(metastoreContext, database);
}

Example 7 with HdfsContext

use of com.facebook.presto.hive.HdfsContext in project presto by prestodb.

the class IcebergUtil method getHiveIcebergTable.

public static Table getHiveIcebergTable(ExtendedHiveMetastore metastore, HdfsEnvironment hdfsEnvironment, ConnectorSession session, SchemaTableName table) {
    HdfsContext hdfsContext = new HdfsContext(session, table.getSchemaName(), table.getTableName());
    TableOperations operations = new HiveTableOperations(metastore, new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), Optional.empty(), false, HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER), hdfsEnvironment, hdfsContext, table.getSchemaName(), table.getTableName());
    return new BaseTable(operations, quotedTableName(table));
}

Example 8 with HdfsContext

use of com.facebook.presto.hive.HdfsContext in project presto by prestodb.

the class IcebergPageSourceProvider method createPageSource.

@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit connectorSplit, ConnectorTableLayoutHandle layout, List<ColumnHandle> columns, SplitContext splitContext) {
    IcebergSplit split = (IcebergSplit) connectorSplit;
    IcebergTableLayoutHandle icebergLayout = (IcebergTableLayoutHandle) layout;
    IcebergTableHandle table = icebergLayout.getTable();
    List<IcebergColumnHandle> icebergColumns = columns.stream().map(IcebergColumnHandle.class::cast).collect(toImmutableList());
    Map<Integer, String> partitionKeys = split.getPartitionKeys();
    List<IcebergColumnHandle> regularColumns = columns.stream().map(IcebergColumnHandle.class::cast).filter(column -> !partitionKeys.containsKey(column.getId())).collect(toImmutableList());
    // TODO: pushdownFilter for icebergLayout
    HdfsContext hdfsContext = new HdfsContext(session, table.getSchemaName(), table.getTableName());
    ConnectorPageSource dataPageSource = createDataPageSource(session, hdfsContext, new Path(split.getPath()), split.getStart(), split.getLength(), split.getFileFormat(), table.getSchemaTableName(), regularColumns, table.getPredicate(), splitContext.isCacheable());
    return new IcebergPageSource(icebergColumns, partitionKeys, dataPageSource, session.getSqlFunctionProperties().getTimeZoneKey());
}

Example 9 with HdfsContext

use of com.facebook.presto.hive.HdfsContext in project presto by prestodb.

the class IcebergPageSink method createWriter.

private WriteContext createWriter(Optional<PartitionData> partitionData) {
    String fileName = fileFormat.addExtension(randomUUID().toString());
    Path outputPath = partitionData.map(partition -> new Path(locationProvider.newDataLocation(partitionSpec, partition, fileName))).orElse(new Path(locationProvider.newDataLocation(fileName)));
    IcebergFileWriter writer = fileWriterFactory.createFileWriter(outputPath, outputSchema, jobConf, session, hdfsContext, fileFormat);
    return new WriteContext(writer, outputPath, partitionData);
}

Example 10 with HdfsContext

use of com.facebook.presto.hive.HdfsContext in project presto by prestodb.

the class IcebergFileWriterFactory method createParquetWriter.

private IcebergFileWriter createParquetWriter(Path outputPath, Schema icebergSchema, JobConf jobConf, ConnectorSession session, HdfsContext hdfsContext) {
    List<String> fileColumnNames = icebergSchema.columns().stream().map(Types.NestedField::name).collect(toImmutableList());
    List<Type> fileColumnTypes = icebergSchema.columns().stream().map(column -> toPrestoType(column.type(), typeManager)).collect(toImmutableList());
    try {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(session.getUser(), outputPath, jobConf);
        Callable<Void> rollbackAction = () -> {
            fileSystem.delete(outputPath, false);
            return null;
        };
        ParquetWriterOptions parquetWriterOptions = ParquetWriterOptions.builder().setMaxPageSize(getParquetWriterPageSize(session)).setMaxPageSize(getParquetWriterBlockSize(session)).build();
        return new IcebergParquetFileWriter(hdfsEnvironment.doAs(session.getUser(), () -> fileSystem.create(outputPath)), rollbackAction, fileColumnNames, fileColumnTypes, convert(icebergSchema, "table"), makeTypeMap(fileColumnTypes, fileColumnNames), parquetWriterOptions, IntStream.range(0, fileColumnNames.size()).toArray(), getCompressionCodec(session).getParquetCompressionCodec().get(), outputPath, hdfsEnvironment, hdfsContext);
    } catch (IOException e) {
        throw new PrestoException(ICEBERG_WRITER_OPEN_ERROR, "Error creating Parquet file", e);
    }
}