Examples with HdfsContext com.facebook.presto.hive.HdfsContext used on opensource projects

Example 16 with HdfsContext

use of com.facebook.presto.hive.HdfsContext in project presto by prestodb.

the class SemiTransactionalHiveMetastore method finishInsertIntoExistingTable.

public synchronized void finishInsertIntoExistingTable(ConnectorSession session, String databaseName, String tableName, Path currentLocation, List<String> fileNames, PartitionStatistics statisticsUpdate) {
    // Data can only be inserted into partitions and unpartitioned tables. They can never be inserted into a partitioned table.
    // Therefore, this method assumes that the table is unpartitioned.
    setShared();
    SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName);
    Action<TableAndMore> oldTableAction = tableActions.get(schemaTableName);
    MetastoreContext metastoreContext = new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), isUserDefinedTypeEncodingEnabled(session), columnConverterProvider);
    if (oldTableAction == null || oldTableAction.getData().getTable().getTableType().equals(TEMPORARY_TABLE)) {
        Table table = getTable(metastoreContext, databaseName, tableName).orElseThrow(() -> new TableNotFoundException(schemaTableName));
        PartitionStatistics currentStatistics = getTableStatistics(metastoreContext, databaseName, tableName);
        HdfsContext context = new HdfsContext(session, databaseName, tableName, table.getStorage().getLocation(), false);
        tableActions.put(schemaTableName, new Action<>(ActionType.INSERT_EXISTING, new TableAndMore(table, Optional.empty(), Optional.of(currentLocation), Optional.of(fileNames), false, merge(currentStatistics, statisticsUpdate), statisticsUpdate), context));
        return;
    }
    switch(oldTableAction.getType()) {
        case DROP:
            throw new TableNotFoundException(schemaTableName);
        case ADD:
        case ALTER:
        case INSERT_EXISTING:
            throw new UnsupportedOperationException("Inserting into an unpartitioned table that were added, altered, or inserted into in the same transaction is not supported");
        default:
            throw new IllegalStateException("Unknown action type");
    }
}

Example 17 with HdfsContext

use of com.facebook.presto.hive.HdfsContext in project presto by prestodb.

the class DeltaPageSourceProvider method createPageSource.

@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, ConnectorTableLayoutHandle layout, List<ColumnHandle> columns, SplitContext splitContext) {
    DeltaSplit deltaSplit = (DeltaSplit) split;
    DeltaTableLayoutHandle deltaTableLayoutHandle = (DeltaTableLayoutHandle) layout;
    DeltaTableHandle deltaTableHandle = deltaTableLayoutHandle.getTable();
    HdfsContext hdfsContext = new HdfsContext(session, deltaSplit.getSchema(), deltaSplit.getTable(), deltaSplit.getFilePath(), false);
    Path filePath = new Path(deltaSplit.getFilePath());
    List<DeltaColumnHandle> deltaColumnHandles = columns.stream().map(DeltaColumnHandle.class::cast).collect(Collectors.toList());
    List<DeltaColumnHandle> regularColumnHandles = deltaColumnHandles.stream().filter(columnHandle -> columnHandle.getColumnType() != PARTITION).collect(Collectors.toList());
    ConnectorPageSource dataPageSource = createParquetPageSource(hdfsEnvironment, session.getUser(), hdfsEnvironment.getConfiguration(hdfsContext, filePath), filePath, deltaSplit.getStart(), deltaSplit.getLength(), deltaSplit.getFileSize(), regularColumnHandles, deltaTableHandle.toSchemaTableName(), getParquetMaxReadBlockSize(session), isParquetBatchReadsEnabled(session), isParquetBatchReaderVerificationEnabled(session), typeManager, deltaTableLayoutHandle.getPredicate(), fileFormatDataSourceStats, false);
    return new DeltaPageSource(deltaColumnHandles, convertPartitionValues(deltaColumnHandles, deltaSplit.getPartitionValues()), dataPageSource);
}

Example 18 with HdfsContext

use of com.facebook.presto.hive.HdfsContext in project presto by prestodb.

the class DeltaClient method loadDeltaTableLog.

private Optional<DeltaLog> loadDeltaTableLog(ConnectorSession session, Path tableLocation, SchemaTableName schemaTableName) {
    try {
        HdfsContext hdfsContext = new HdfsContext(session, schemaTableName.getSchemaName(), schemaTableName.getTableName(), tableLocation.toString(), false);
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(hdfsContext, tableLocation);
        if (!fileSystem.isDirectory(tableLocation)) {
            return Optional.empty();
        }
        return Optional.of(DeltaLog.forTable(hdfsEnvironment.getConfiguration(hdfsContext, tableLocation), tableLocation));
    } catch (IOException ioException) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, "Failed to load Delta table: " + ioException.getMessage(), ioException);
    }
}

Example 19 with HdfsContext

use of com.facebook.presto.hive.HdfsContext in project presto by prestodb.

the class RaptorPageSinkProvider method createPageSink.

@Override
public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorInsertTableHandle tableHandle, PageSinkContext pageSinkContext) {
    checkArgument(!pageSinkContext.isCommitRequired(), "Raptor connector does not support page sink commit");
    RaptorInsertTableHandle handle = (RaptorInsertTableHandle) tableHandle;
    return new RaptorPageSink(new HdfsContext(session), pageSorter, storageManager, temporalFunction, handle.getTransactionId(), toColumnIds(handle.getColumnHandles()), handle.getColumnTypes(), toColumnIds(handle.getSortColumnHandles()), handle.getSortOrders(), handle.getBucketCount(), toColumnIds(handle.getBucketColumnHandles()), handle.getTemporalColumnHandle(), getWriterMaxBufferSize(session), maxAllowedFilesPerWriter);
}