Examples with CommitInfoEntry io.trino.plugin.deltalake.transactionlog.CommitInfoEntry used on opensource projects

Example 1 with CommitInfoEntry

use of io.trino.plugin.deltalake.transactionlog.CommitInfoEntry in project trino by trinodb.

the class DeltaLakeMetadata method appendInitialTableEntries.

private static void appendInitialTableEntries(TransactionLogWriter transactionLogWriter, List<DeltaLakeColumnHandle> columns, List<String> partitionColumnNames, Map<String, String> configuration, String operation, ConnectorSession session, String nodeVersion, String nodeId) {
    long createdTime = System.currentTimeMillis();
    transactionLogWriter.appendCommitInfoEntry(new CommitInfoEntry(0, createdTime, session.getUser(), session.getUser(), operation, ImmutableMap.of("queryId", session.getQueryId()), null, null, "trino-" + nodeVersion + "-" + nodeId, 0, ISOLATION_LEVEL, true));
    transactionLogWriter.appendProtocolEntry(new ProtocolEntry(READER_VERSION, WRITER_VERSION));
    transactionLogWriter.appendMetadataEntry(new MetadataEntry(randomUUID().toString(), null, null, new Format("parquet", ImmutableMap.of()), serializeSchemaAsJson(columns), partitionColumnNames, ImmutableMap.copyOf(configuration), createdTime));
}

Example 2 with CommitInfoEntry

use of io.trino.plugin.deltalake.transactionlog.CommitInfoEntry in project trino by trinodb.

the class CheckpointEntryIterator method buildCommitInfoEntry.

private DeltaLakeTransactionLogEntry buildCommitInfoEntry(ConnectorSession session, Block block, int pagePosition) {
    log.debug("Building commitInfo entry from %s pagePosition %d", block, pagePosition);
    if (block.isNull(pagePosition)) {
        return null;
    }
    int commitInfoFields = 12;
    int jobFields = 5;
    int notebookFields = 1;
    Block commitInfoEntryBlock = block.getObject(pagePosition, Block.class);
    log.debug("Block %s has %s fields", block, commitInfoEntryBlock.getPositionCount());
    if (commitInfoEntryBlock.getPositionCount() != commitInfoFields) {
        throw new TrinoException(DELTA_LAKE_INVALID_SCHEMA, format("Expected block %s to have %d children, but found %s", block, commitInfoFields, commitInfoEntryBlock.getPositionCount()));
    }
    Block jobBlock = commitInfoEntryBlock.getObject(6, Block.class);
    if (jobBlock.getPositionCount() != jobFields) {
        throw new TrinoException(DELTA_LAKE_INVALID_SCHEMA, format("Expected block %s to have %d children, but found %s", jobBlock, jobFields, jobBlock.getPositionCount()));
    }
    Block notebookBlock = commitInfoEntryBlock.getObject(7, Block.class);
    if (notebookBlock.getPositionCount() != notebookFields) {
        throw new TrinoException(DELTA_LAKE_INVALID_SCHEMA, format("Expected block %s to have %d children, but found %s", notebookBlock, notebookFields, notebookBlock.getPositionCount()));
    }
    CommitInfoEntry result = new CommitInfoEntry(getLong(commitInfoEntryBlock, 0), getLong(commitInfoEntryBlock, 1), getString(commitInfoEntryBlock, 2), getString(commitInfoEntryBlock, 3), getString(commitInfoEntryBlock, 4), getMap(commitInfoEntryBlock, 5), new CommitInfoEntry.Job(getString(jobBlock, 0), getString(jobBlock, 1), getString(jobBlock, 2), getString(jobBlock, 3), getString(jobBlock, 4)), new CommitInfoEntry.Notebook(getString(notebookBlock, 0)), getString(commitInfoEntryBlock, 8), getLong(commitInfoEntryBlock, 9), getString(commitInfoEntryBlock, 10), getByte(commitInfoEntryBlock, 11) != 0);
    log.debug("Result: %s", result);
    return DeltaLakeTransactionLogEntry.commitInfoEntry(result);
}

Example 3 with CommitInfoEntry

use of io.trino.plugin.deltalake.transactionlog.CommitInfoEntry in project trino by trinodb.

the class TransactionLogWriter method flush.

public void flush() throws IOException {
    checkState(commitInfoEntry.isPresent(), "commitInfo not set");
    Path transactionLogLocation = getTransactionLogDir(new Path(tableLocation));
    CommitInfoEntry commitInfo = requireNonNull(commitInfoEntry.get().getCommitInfo(), "commitInfoEntry.get().getCommitInfo() is null");
    Path logEntry = getTransactionLogJsonEntryPath(transactionLogLocation, commitInfo.getVersion());
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    writeEntry(bos, commitInfoEntry.get());
    for (DeltaLakeTransactionLogEntry entry : entries) {
        writeEntry(bos, entry);
    }
    String clusterId = commitInfoEntry.get().getCommitInfo().getClusterId();
    logSynchronizer.write(session, clusterId, logEntry, bos.toByteArray());
}

Example 4 with CommitInfoEntry

use of io.trino.plugin.deltalake.transactionlog.CommitInfoEntry in project trino by trinodb.

the class DeltaLakeMetadata method finishOptimize.

private void finishOptimize(ConnectorSession session, DeltaLakeTableExecuteHandle executeHandle, Collection<Slice> fragments, List<Object> splitSourceInfo) {
    DeltaTableOptimizeHandle optimizeHandle = (DeltaTableOptimizeHandle) executeHandle.getProcedureHandle();
    long readVersion = optimizeHandle.getCurrentVersion().orElseThrow(() -> new IllegalArgumentException("currentVersion not set"));
    Optional<Long> checkpointInterval = optimizeHandle.getMetadataEntry().getCheckpointInterval();
    String tableLocation = executeHandle.getTableLocation();
    // paths to be deleted
    Set<Path> scannedPaths = splitSourceInfo.stream().map(file -> new Path((String) file)).collect(toImmutableSet());
    // files to be added
    List<DataFileInfo> dataFileInfos = fragments.stream().map(Slice::getBytes).map(dataFileInfoCodec::fromJson).collect(toImmutableList());
    boolean writeCommitted = false;
    try {
        TransactionLogWriter transactionLogWriter = transactionLogWriterFactory.newWriter(session, tableLocation);
        long createdTime = Instant.now().toEpochMilli();
        long commitVersion = readVersion + 1;
        transactionLogWriter.appendCommitInfoEntry(new CommitInfoEntry(commitVersion, createdTime, session.getUser(), session.getUser(), OPTIMIZE_OPERATION, ImmutableMap.of("queryId", session.getQueryId()), null, null, "trino-" + nodeVersion + "-" + nodeId, readVersion, ISOLATION_LEVEL, true));
        // TODO: Delta writes another field "operationMetrics" that I haven't
        // seen before. It contains delete/update metrics. Investigate/include it.
        long writeTimestamp = Instant.now().toEpochMilli();
        for (Path scannedPath : scannedPaths) {
            String relativePath = new Path(tableLocation).toUri().relativize(scannedPath.toUri()).toString();
            transactionLogWriter.appendRemoveFileEntry(new RemoveFileEntry(relativePath, writeTimestamp, false));
        }
        // Note: during writes we want to preserve original case of partition columns
        List<String> partitionColumns = optimizeHandle.getMetadataEntry().getOriginalPartitionColumns();
        appendAddFileEntries(transactionLogWriter, dataFileInfos, partitionColumns, false);
        transactionLogWriter.flush();
        writeCommitted = true;
        writeCheckpointIfNeeded(session, executeHandle.getSchemaTableName(), checkpointInterval, commitVersion);
    } catch (Exception e) {
        if (!writeCommitted) {
            // TODO perhaps it should happen in a background thread
            cleanupFailedWrite(session, tableLocation, dataFileInfos);
        }
        throw new TrinoException(DELTA_LAKE_BAD_WRITE, "Failed to write Delta Lake transaction log entry", e);
    }
}

Example 5 with CommitInfoEntry

use of io.trino.plugin.deltalake.transactionlog.CommitInfoEntry in project trino by trinodb.

the class DeltaLakeMetadata method finishWrite.

private void finishWrite(ConnectorSession session, ConnectorTableHandle tableHandle, Collection<Slice> fragments) {
    DeltaLakeTableHandle handle = (DeltaLakeTableHandle) tableHandle;
    List<DeltaLakeUpdateResult> updateResults = fragments.stream().map(Slice::getBytes).map(deleteResultJsonCodec::fromJson).collect(toImmutableList());
    String tableLocation = metastore.getTableLocation(handle.getSchemaTableName(), session);
    DeltaLakeTableHandle.WriteType writeType = handle.getWriteType().orElseThrow();
    String operation;
    switch(writeType) {
        case DELETE:
            operation = DELETE_OPERATION;
            break;
        case UPDATE:
            operation = UPDATE_OPERATION;
            break;
        default:
            throw new TrinoException(NOT_SUPPORTED, "Unsupported write type: " + writeType);
    }
    boolean writeCommitted = false;
    try {
        TransactionLogWriter transactionLogWriter = transactionLogWriterFactory.newWriter(session, tableLocation);
        long createdTime = Instant.now().toEpochMilli();
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(new HdfsContext(session), new Path(tableLocation));
        long commitVersion = getMandatoryCurrentVersion(fileSystem, new Path(tableLocation)) + 1;
        if (commitVersion != handle.getReadVersion() + 1) {
            throw new TransactionConflictException(format("Conflicting concurrent writes found. Expected transaction log version: %s, actual version: %s", handle.getReadVersion(), commitVersion - 1));
        }
        Optional<Long> checkpointInterval = handle.getMetadataEntry().getCheckpointInterval();
        transactionLogWriter.appendCommitInfoEntry(new CommitInfoEntry(commitVersion, createdTime, session.getUser(), session.getUser(), operation, ImmutableMap.of("queryId", session.getQueryId()), null, null, "trino-" + nodeVersion + "-" + nodeId, // TODO Insert fills this in with, probably should do so here too
        0, ISOLATION_LEVEL, true));
        // TODO: Delta writes another field "operationMetrics" that I haven't
        // seen before. It contains delete/update metrics. Investigate/include it.
        long writeTimestamp = Instant.now().toEpochMilli();
        for (DeltaLakeUpdateResult updateResult : updateResults) {
            transactionLogWriter.appendRemoveFileEntry(new RemoveFileEntry(updateResult.getOldFile(), writeTimestamp, true));
        }
        appendAddFileEntries(transactionLogWriter, updateResults.stream().map(DeltaLakeUpdateResult::getNewFile).filter(Optional::isPresent).map(Optional::get).collect(toImmutableList()), handle.getMetadataEntry().getOriginalPartitionColumns(), true);
        transactionLogWriter.flush();
        writeCommitted = true;
        writeCheckpointIfNeeded(session, new SchemaTableName(handle.getSchemaName(), handle.getTableName()), checkpointInterval, commitVersion);
    } catch (Exception e) {
        if (!writeCommitted) {
            // TODO perhaps it should happen in a background thread
            cleanupFailedWrite(session, tableLocation, updateResults.stream().map(DeltaLakeUpdateResult::getNewFile).filter(Optional::isPresent).map(Optional::get).collect(toImmutableList()));
        }
        throw new TrinoException(DELTA_LAKE_BAD_WRITE, "Failed to write Delta Lake transaction log entry", e);
    }
}