Examples with WriteInfo io.trino.plugin.hive.LocationService.WriteInfo used on opensource projects

Example 6 with WriteInfo

use of io.trino.plugin.hive.LocationService.WriteInfo in project trino by trinodb.

the class CreateEmptyPartitionProcedure method doCreateEmptyPartition.

private void doCreateEmptyPartition(ConnectorSession session, ConnectorAccessControl accessControl, String schemaName, String tableName, List<String> partitionColumnNames, List<String> partitionValues) {
    TransactionalMetadata hiveMetadata = hiveMetadataFactory.create(session.getIdentity(), true);
    HiveTableHandle tableHandle = (HiveTableHandle) hiveMetadata.getTableHandle(session, new SchemaTableName(schemaName, tableName));
    if (tableHandle == null) {
        throw new TrinoException(INVALID_PROCEDURE_ARGUMENT, format("Table '%s' does not exist", new SchemaTableName(schemaName, tableName)));
    }
    accessControl.checkCanInsertIntoTable(null, new SchemaTableName(schemaName, tableName));
    List<String> actualPartitionColumnNames = tableHandle.getPartitionColumns().stream().map(HiveColumnHandle::getName).collect(toImmutableList());
    if (!Objects.equals(partitionColumnNames, actualPartitionColumnNames)) {
        throw new TrinoException(INVALID_PROCEDURE_ARGUMENT, "Provided partition column names do not match actual partition column names: " + actualPartitionColumnNames);
    }
    HiveMetastoreClosure metastore = hiveMetadata.getMetastore().unsafeGetRawHiveMetastoreClosure();
    if (metastore.getPartition(schemaName, tableName, partitionValues).isPresent()) {
        throw new TrinoException(ALREADY_EXISTS, "Partition already exists");
    }
    HiveInsertTableHandle hiveInsertTableHandle = (HiveInsertTableHandle) hiveMetadata.beginInsert(session, tableHandle, ImmutableList.of(), NO_RETRIES);
    String partitionName = FileUtils.makePartName(actualPartitionColumnNames, partitionValues);
    WriteInfo writeInfo = locationService.getPartitionWriteInfo(hiveInsertTableHandle.getLocationHandle(), Optional.empty(), partitionName);
    Slice serializedPartitionUpdate = Slices.wrappedBuffer(partitionUpdateJsonCodec.toJsonBytes(new PartitionUpdate(partitionName, UpdateMode.NEW, writeInfo.getWritePath(), writeInfo.getTargetPath(), ImmutableList.of(), 0, 0, 0)));
    hiveMetadata.finishInsert(session, hiveInsertTableHandle, ImmutableList.of(serializedPartitionUpdate), ImmutableList.of());
    hiveMetadata.commit();
}

Example 7 with WriteInfo

use of io.trino.plugin.hive.LocationService.WriteInfo in project trino by trinodb.

the class TestHiveLocationService method testGetTableWriteInfoAppend.

@Test
public void testGetTableWriteInfoAppend() {
    assertThat(locationHandle(STAGE_AND_MOVE_TO_TARGET_DIRECTORY), false).producesWriteInfo(new WriteInfo(new Path("/target"), new Path("/write"), STAGE_AND_MOVE_TO_TARGET_DIRECTORY));
    assertThat(locationHandle(DIRECT_TO_TARGET_EXISTING_DIRECTORY, "/target", "/target"), false).producesWriteInfo(new WriteInfo(new Path("/target"), new Path("/target"), DIRECT_TO_TARGET_EXISTING_DIRECTORY));
    assertThat(locationHandle(DIRECT_TO_TARGET_NEW_DIRECTORY, "/target", "/target"), false).producesWriteInfo(new WriteInfo(new Path("/target"), new Path("/target"), DIRECT_TO_TARGET_NEW_DIRECTORY));
}

Example 8 with WriteInfo

use of io.trino.plugin.hive.LocationService.WriteInfo in project trino by trinodb.

the class AbstractTestHive method getStagingPathRoot.

// These are protected so extensions to the hive connector can replace the handle classes
protected Path getStagingPathRoot(ConnectorInsertTableHandle insertTableHandle) {
    HiveInsertTableHandle handle = (HiveInsertTableHandle) insertTableHandle;
    WriteInfo writeInfo = getLocationService().getQueryWriteInfo(handle.getLocationHandle());
    if (writeInfo.getWriteMode() != STAGE_AND_MOVE_TO_TARGET_DIRECTORY) {
        throw new AssertionError("writeMode is not STAGE_AND_MOVE_TO_TARGET_DIRECTORY");
    }
    return writeInfo.getWritePath();
}

Example 9 with WriteInfo

use of io.trino.plugin.hive.LocationService.WriteInfo in project trino by trinodb.

the class HiveMetadata method beginCreateTable.

@Override
public HiveOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorTableLayout> layout, RetryMode retryMode) {
    Optional<Path> externalLocation = Optional.ofNullable(getExternalLocation(tableMetadata.getProperties())).map(HiveMetadata::getExternalLocationAsPath);
    if (!createsOfNonManagedTablesEnabled && externalLocation.isPresent()) {
        throw new TrinoException(NOT_SUPPORTED, "Creating non-managed Hive tables is disabled");
    }
    if (!writesToNonManagedTablesEnabled && externalLocation.isPresent()) {
        throw new TrinoException(NOT_SUPPORTED, "Writes to non-managed Hive tables is disabled");
    }
    boolean isTransactional = isTransactional(tableMetadata.getProperties()).orElse(false);
    if (isTransactional && externalLocation.isEmpty() && isDelegateTransactionalManagedTableLocationToMetastore(session)) {
        throw new TrinoException(NOT_SUPPORTED, "CREATE TABLE AS is not supported for transactional tables without explicit location if location determining is delegated to metastore");
    }
    if (isTransactional && retryMode != NO_RETRIES) {
        throw new TrinoException(NOT_SUPPORTED, "CREATE TABLE AS is not supported for transactional tables with query retries enabled");
    }
    if (getAvroSchemaUrl(tableMetadata.getProperties()) != null) {
        throw new TrinoException(NOT_SUPPORTED, "CREATE TABLE AS not supported when Avro schema url is set");
    }
    getHeaderSkipCount(tableMetadata.getProperties()).ifPresent(headerSkipCount -> {
        if (headerSkipCount > 1) {
            throw new TrinoException(NOT_SUPPORTED, format("Creating Hive table with data with value of %s property greater than 1 is not supported", SKIP_HEADER_COUNT_KEY));
        }
    });
    getFooterSkipCount(tableMetadata.getProperties()).ifPresent(footerSkipCount -> {
        if (footerSkipCount > 0) {
            throw new TrinoException(NOT_SUPPORTED, format("Creating Hive table with data with value of %s property greater than 0 is not supported", SKIP_FOOTER_COUNT_KEY));
        }
    });
    HiveStorageFormat tableStorageFormat = getHiveStorageFormat(tableMetadata.getProperties());
    List<String> partitionedBy = getPartitionedBy(tableMetadata.getProperties());
    Optional<HiveBucketProperty> bucketProperty = getBucketProperty(tableMetadata.getProperties());
    // get the root directory for the database
    SchemaTableName schemaTableName = tableMetadata.getTable();
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();
    Map<String, String> tableProperties = getEmptyTableProperties(tableMetadata, bucketProperty, new HdfsContext(session));
    List<HiveColumnHandle> columnHandles = getColumnHandles(tableMetadata, ImmutableSet.copyOf(partitionedBy));
    HiveStorageFormat partitionStorageFormat = isRespectTableFormat(session) ? tableStorageFormat : getHiveStorageFormat(session);
    // unpartitioned tables ignore the partition storage format
    HiveStorageFormat actualStorageFormat = partitionedBy.isEmpty() ? tableStorageFormat : partitionStorageFormat;
    actualStorageFormat.validateColumns(columnHandles);
    Map<String, HiveColumnHandle> columnHandlesByName = Maps.uniqueIndex(columnHandles, HiveColumnHandle::getName);
    List<Column> partitionColumns = partitionedBy.stream().map(columnHandlesByName::get).map(HiveColumnHandle::toMetastoreColumn).collect(toImmutableList());
    checkPartitionTypesSupported(partitionColumns);
    LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, externalLocation);
    AcidTransaction transaction = isTransactional ? forCreateTable() : NO_ACID_TRANSACTION;
    HiveOutputTableHandle result = new HiveOutputTableHandle(schemaName, tableName, columnHandles, metastore.generatePageSinkMetadata(schemaTableName), locationHandle, tableStorageFormat, partitionStorageFormat, partitionedBy, bucketProperty, session.getUser(), tableProperties, transaction, externalLocation.isPresent(), retryMode != NO_RETRIES);
    WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
    metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), schemaTableName);
    return result;
}

Example 10 with WriteInfo

use of io.trino.plugin.hive.LocationService.WriteInfo in project trino by trinodb.

the class HiveMetadata method beginUpdate.

@Override
public ConnectorTableHandle beginUpdate(ConnectorSession session, ConnectorTableHandle tableHandle, List<ColumnHandle> updatedColumns, RetryMode retryMode) {
    HiveTableHandle hiveTableHandle = (HiveTableHandle) tableHandle;
    SchemaTableName tableName = hiveTableHandle.getSchemaTableName();
    Table table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    if (!isFullAcidTable(table.getParameters())) {
        throw new TrinoException(NOT_SUPPORTED, "Hive update is only supported for ACID transactional tables");
    }
    if (!autoCommit) {
        throw new TrinoException(NOT_SUPPORTED, "Updating transactional tables is not supported in explicit transactions (use autocommit mode)");
    }
    if (isSparkBucketedTable(table)) {
        throw new TrinoException(NOT_SUPPORTED, "Updating Spark bucketed tables is not supported");
    }
    // Verify that none of the updated columns are partition columns or bucket columns
    Set<String> updatedColumnNames = updatedColumns.stream().map(handle -> ((HiveColumnHandle) handle).getName()).collect(toImmutableSet());
    Set<String> partitionColumnNames = table.getPartitionColumns().stream().map(Column::getName).collect(toImmutableSet());
    if (!intersection(updatedColumnNames, partitionColumnNames).isEmpty()) {
        throw new TrinoException(NOT_SUPPORTED, "Updating Hive table partition columns is not supported");
    }
    hiveTableHandle.getBucketHandle().ifPresent(handle -> {
        Set<String> bucketColumnNames = handle.getColumns().stream().map(HiveColumnHandle::getName).collect(toImmutableSet());
        if (!intersection(updatedColumnNames, bucketColumnNames).isEmpty()) {
            throw new TrinoException(NOT_SUPPORTED, "Updating Hive table bucket columns is not supported");
        }
    });
    checkTableIsWritable(table, writesToNonManagedTablesEnabled);
    for (Column column : table.getDataColumns()) {
        if (!isWritableType(column.getType())) {
            throw new TrinoException(NOT_SUPPORTED, format("Updating a Hive table with column type %s not supported", column.getType()));
        }
    }
    List<HiveColumnHandle> allDataColumns = getRegularColumnHandles(table, typeManager, getTimestampPrecision(session)).stream().filter(columnHandle -> !columnHandle.isHidden()).collect(toImmutableList());
    List<HiveColumnHandle> hiveUpdatedColumns = updatedColumns.stream().map(HiveColumnHandle.class::cast).collect(toImmutableList());
    if (table.getParameters().containsKey(SKIP_HEADER_COUNT_KEY)) {
        throw new TrinoException(NOT_SUPPORTED, format("Updating a Hive table with %s property not supported", SKIP_HEADER_COUNT_KEY));
    }
    if (table.getParameters().containsKey(SKIP_FOOTER_COUNT_KEY)) {
        throw new TrinoException(NOT_SUPPORTED, format("Updating a Hive table with %s property not supported", SKIP_FOOTER_COUNT_KEY));
    }
    if (retryMode != NO_RETRIES) {
        throw new TrinoException(NOT_SUPPORTED, "Updating a Hive tables is not supported with query retries enabled");
    }
    LocationHandle locationHandle = locationService.forExistingTable(metastore, session, table);
    HiveUpdateProcessor updateProcessor = new HiveUpdateProcessor(allDataColumns, hiveUpdatedColumns);
    AcidTransaction transaction = metastore.beginUpdate(session, table, updateProcessor);
    HiveTableHandle updateHandle = hiveTableHandle.withTransaction(transaction);
    WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
    metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), tableName);
    return updateHandle;
}