Examples with AcidTransaction io.trino.plugin.hive.acid.AcidTransaction used on opensource projects

Example 1 with AcidTransaction

use of io.trino.plugin.hive.acid.AcidTransaction in project trino by trinodb.

the class HiveMetadata method beginDelete.

@Override
public ConnectorTableHandle beginDelete(ConnectorSession session, ConnectorTableHandle tableHandle, RetryMode retryMode) {
    HiveTableHandle handle = (HiveTableHandle) tableHandle;
    SchemaTableName tableName = handle.getSchemaTableName();
    Table table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    ensureTableSupportsDelete(table);
    if (retryMode != NO_RETRIES) {
        throw new TrinoException(NOT_SUPPORTED, "Deleting from Hive tables is not supported with query retries enabled");
    }
    if (!autoCommit) {
        throw new TrinoException(NOT_SUPPORTED, "Deleting from Hive transactional tables is not supported in explicit transactions (use autocommit mode)");
    }
    if (isSparkBucketedTable(table)) {
        throw new TrinoException(NOT_SUPPORTED, "Deleting from Spark bucketed tables is not supported");
    }
    LocationHandle locationHandle = locationService.forExistingTable(metastore, session, table);
    AcidTransaction transaction = metastore.beginDelete(session, table);
    WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
    metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), handle.getSchemaTableName());
    return handle.withTransaction(transaction);
}

Example 2 with AcidTransaction

use of io.trino.plugin.hive.acid.AcidTransaction in project trino by trinodb.

the class HiveMetadata method beginInsert.

@Override
public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle, List<ColumnHandle> columns, RetryMode retryMode) {
    SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName();
    Table table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    checkTableIsWritable(table, writesToNonManagedTablesEnabled);
    for (Column column : table.getDataColumns()) {
        if (!isWritableType(column.getType())) {
            throw new TrinoException(NOT_SUPPORTED, format("Inserting into Hive table %s with column type %s not supported", tableName, column.getType()));
        }
    }
    boolean isTransactional = isTransactionalTable(table.getParameters());
    if (isTransactional && retryMode != NO_RETRIES) {
        throw new TrinoException(NOT_SUPPORTED, "Inserting into Hive transactional tables is not supported with query retries enabled");
    }
    if (isTransactional && !autoCommit) {
        throw new TrinoException(NOT_SUPPORTED, "Inserting into Hive transactional tables is not supported in explicit transactions (use autocommit mode)");
    }
    if (isSparkBucketedTable(table)) {
        throw new TrinoException(NOT_SUPPORTED, "Inserting into Spark bucketed tables is not supported");
    }
    List<HiveColumnHandle> handles = hiveColumnHandles(table, typeManager, getTimestampPrecision(session)).stream().filter(columnHandle -> !columnHandle.isHidden()).collect(toImmutableList());
    HiveStorageFormat tableStorageFormat = extractHiveStorageFormat(table);
    Optional.ofNullable(table.getParameters().get(SKIP_HEADER_COUNT_KEY)).map(Integer::parseInt).ifPresent(headerSkipCount -> {
        if (headerSkipCount > 1) {
            throw new TrinoException(NOT_SUPPORTED, format("Inserting into Hive table with value of %s property greater than 1 is not supported", SKIP_HEADER_COUNT_KEY));
        }
    });
    if (table.getParameters().containsKey(SKIP_FOOTER_COUNT_KEY)) {
        throw new TrinoException(NOT_SUPPORTED, format("Inserting into Hive table with %s property not supported", SKIP_FOOTER_COUNT_KEY));
    }
    LocationHandle locationHandle = locationService.forExistingTable(metastore, session, table);
    AcidTransaction transaction = isTransactional ? metastore.beginInsert(session, table) : NO_ACID_TRANSACTION;
    HiveInsertTableHandle result = new HiveInsertTableHandle(tableName.getSchemaName(), tableName.getTableName(), handles, metastore.generatePageSinkMetadata(tableName), locationHandle, table.getStorage().getBucketProperty(), tableStorageFormat, isRespectTableFormat(session) ? tableStorageFormat : getHiveStorageFormat(session), transaction, retryMode != NO_RETRIES);
    WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
    if (getInsertExistingPartitionsBehavior(session) == InsertExistingPartitionsBehavior.OVERWRITE && writeInfo.getWriteMode() == DIRECT_TO_TARGET_EXISTING_DIRECTORY) {
        if (isTransactional) {
            throw new TrinoException(NOT_SUPPORTED, "Overwriting existing partition in transactional tables doesn't support DIRECT_TO_TARGET_EXISTING_DIRECTORY write mode");
        }
        // Partition overwrite operation is nonatomic thus can't and shouldn't be used in non autocommit context.
        if (!autoCommit) {
            throw new TrinoException(NOT_SUPPORTED, "Overwriting existing partition in non auto commit context doesn't support DIRECT_TO_TARGET_EXISTING_DIRECTORY write mode");
        }
    }
    metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), tableName);
    return result;
}

Example 3 with AcidTransaction

use of io.trino.plugin.hive.acid.AcidTransaction in project trino by trinodb.

the class RcFileFileWriterFactory method createFileWriter.

@Override
public Optional<FileWriter> createFileWriter(Path path, List<String> inputColumnNames, StorageFormat storageFormat, Properties schema, JobConf configuration, ConnectorSession session, OptionalInt bucketNumber, AcidTransaction transaction, boolean useAcidSchema, WriterKind writerKind) {
    if (!RCFileOutputFormat.class.getName().equals(storageFormat.getOutputFormat())) {
        return Optional.empty();
    }
    RcFileEncoding rcFileEncoding;
    if (LazyBinaryColumnarSerDe.class.getName().equals(storageFormat.getSerde())) {
        rcFileEncoding = new BinaryRcFileEncoding(timeZone);
    } else if (ColumnarSerDe.class.getName().equals(storageFormat.getSerde())) {
        rcFileEncoding = createTextVectorEncoding(schema);
    } else {
        return Optional.empty();
    }
    Optional<String> codecName = Optional.ofNullable(configuration.get(FileOutputFormat.COMPRESS_CODEC));
    // existing tables and partitions may have columns in a different order than the writer is providing, so build
    // an index to rearrange columns in the proper order
    List<String> fileColumnNames = getColumnNames(schema);
    List<Type> fileColumnTypes = getColumnTypes(schema).stream().map(hiveType -> hiveType.getType(typeManager, getTimestampPrecision(session))).collect(toList());
    int[] fileInputColumnIndexes = fileColumnNames.stream().mapToInt(inputColumnNames::indexOf).toArray();
    try {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(session.getIdentity(), path, configuration);
        OutputStream outputStream = fileSystem.create(path, false);
        Optional<Supplier<RcFileDataSource>> validationInputFactory = Optional.empty();
        if (isRcfileOptimizedWriterValidate(session)) {
            validationInputFactory = Optional.of(() -> {
                try {
                    return new HdfsRcFileDataSource(path.toString(), fileSystem.open(path), fileSystem.getFileStatus(path).getLen(), stats);
                } catch (IOException e) {
                    throw new TrinoException(HIVE_WRITE_VALIDATION_FAILED, e);
                }
            });
        }
        Callable<Void> rollbackAction = () -> {
            fileSystem.delete(path, false);
            return null;
        };
        return Optional.of(new RcFileFileWriter(outputStream, rollbackAction, rcFileEncoding, fileColumnTypes, codecName, fileInputColumnIndexes, ImmutableMap.<String, String>builder().put(PRESTO_VERSION_NAME, nodeVersion.toString()).put(PRESTO_QUERY_ID_NAME, session.getQueryId()).buildOrThrow(), validationInputFactory));
    } catch (Exception e) {
        throw new TrinoException(HIVE_WRITER_OPEN_ERROR, "Error creating RCFile file", e);
    }
}

Example 4 with AcidTransaction

use of io.trino.plugin.hive.acid.AcidTransaction in project trino by trinodb.

the class SemiTransactionalHiveMetastore method setTableStatistics.

// TODO: Allow updating statistics for 2 tables in the same transaction
public synchronized void setTableStatistics(Table table, PartitionStatistics tableStatistics) {
    AcidTransaction transaction = currentHiveTransaction.isPresent() ? currentHiveTransaction.get().getTransaction() : NO_ACID_TRANSACTION;
    setExclusive((delegate, hdfsEnvironment) -> delegate.updateTableStatistics(table.getDatabaseName(), table.getTableName(), transaction, statistics -> updatePartitionStatistics(statistics, tableStatistics)));
}

Example 5 with AcidTransaction

use of io.trino.plugin.hive.acid.AcidTransaction in project trino by trinodb.

the class SemiTransactionalHiveMetastore method makeHiveTransaction.

private HiveTransaction makeHiveTransaction(ConnectorSession session, Function<Long, AcidTransaction> transactionMaker) {
    String queryId = session.getQueryId();
    long heartbeatInterval = configuredTransactionHeartbeatInterval.map(Duration::toMillis).orElseGet(this::getServerExpectedHeartbeatIntervalMillis);
    // TODO consider adding query id to the owner
    long transactionId = delegate.openTransaction(new AcidTransactionOwner(session.getUser()));
    log.debug("Using hive transaction %s for %s", transactionId, queryId);
    ScheduledFuture<?> heartbeatTask = heartbeatExecutor.scheduleAtFixedRate(() -> delegate.sendTransactionHeartbeat(transactionId), 0, heartbeatInterval, MILLISECONDS);
    AcidTransaction transaction = transactionMaker.apply(transactionId);
    return new HiveTransaction(queryId, transactionId, heartbeatTask, transaction);
}