Examples with HoodieCommitException org.apache.hudi.exception.HoodieCommitException used on opensource projects

Example 11 with HoodieCommitException

use of org.apache.hudi.exception.HoodieCommitException in project hudi by apache.

the class BaseHoodieWriteClient method commitStats.

public boolean commitStats(String instantTime, List<HoodieWriteStat> stats, Option<Map<String, String>> extraMetadata, String commitActionType, Map<String, List<String>> partitionToReplaceFileIds) {
    // Skip the empty commit if not allowed
    if (!config.allowEmptyCommit() && stats.isEmpty()) {
        return true;
    }
    LOG.info("Committing " + instantTime + " action " + commitActionType);
    // Create a Hoodie table which encapsulated the commits and files visible
    HoodieTable table = createTable(config, hadoopConf);
    HoodieCommitMetadata metadata = CommitUtils.buildMetadata(stats, partitionToReplaceFileIds, extraMetadata, operationType, config.getWriteSchema(), commitActionType);
    HoodieInstant inflightInstant = new HoodieInstant(State.INFLIGHT, table.getMetaClient().getCommitActionType(), instantTime);
    HeartbeatUtils.abortIfHeartbeatExpired(instantTime, table, heartbeatClient, config);
    this.txnManager.beginTransaction(Option.of(inflightInstant), lastCompletedTxnAndMetadata.isPresent() ? Option.of(lastCompletedTxnAndMetadata.get().getLeft()) : Option.empty());
    try {
        preCommit(inflightInstant, metadata);
        commit(table, commitActionType, instantTime, metadata, stats);
        postCommit(table, metadata, instantTime, extraMetadata);
        LOG.info("Committed " + instantTime);
        releaseResources();
    } catch (IOException e) {
        throw new HoodieCommitException("Failed to complete commit " + config.getBasePath() + " at time " + instantTime, e);
    } finally {
        this.txnManager.endTransaction(Option.of(inflightInstant));
    }
    // do this outside of lock since compaction, clustering can be time taking and we don't need a lock for the entire execution period
    runTableServicesInline(table, metadata, extraMetadata);
    emitCommitMetrics(instantTime, metadata, commitActionType);
    // callback if needed.
    if (config.writeCommitCallbackOn()) {
        if (null == commitCallback) {
            commitCallback = HoodieCommitCallbackFactory.create(config);
        }
        commitCallback.call(new HoodieWriteCommitCallbackMessage(instantTime, config.getTableName(), config.getBasePath(), stats));
    }
    return true;
}

Example 12 with HoodieCommitException

use of org.apache.hudi.exception.HoodieCommitException in project hudi by apache.

the class HoodieTimelineArchiver method mergeArchiveFiles.

public void mergeArchiveFiles(List<FileStatus> compactCandidate) throws IOException {
    LOG.info("Starting to merge small archive files.");
    Schema wrapperSchema = HoodieArchivedMetaEntry.getClassSchema();
    try {
        List<IndexedRecord> records = new ArrayList<>();
        for (FileStatus fs : compactCandidate) {
            // Read the archived file
            try (HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(metaClient.getFs(), new HoodieLogFile(fs.getPath()), HoodieArchivedMetaEntry.getClassSchema())) {
                // Read the avro blocks
                while (reader.hasNext()) {
                    HoodieAvroDataBlock blk = (HoodieAvroDataBlock) reader.next();
                    blk.getRecordItr().forEachRemaining(records::add);
                    if (records.size() >= this.config.getCommitArchivalBatchSize()) {
                        writeToFile(wrapperSchema, records);
                    }
                }
            }
        }
        writeToFile(wrapperSchema, records);
    } catch (Exception e) {
        throw new HoodieCommitException("Failed to merge small archive files", e);
    } finally {
        writer.close();
    }
    LOG.info("Success to merge small archive files.");
}

Example 13 with HoodieCommitException

use of org.apache.hudi.exception.HoodieCommitException in project hudi by apache.

the class HoodieTimelineArchiver method archive.

public void archive(HoodieEngineContext context, List<HoodieInstant> instants) throws HoodieCommitException {
    try {
        Schema wrapperSchema = HoodieArchivedMetaEntry.getClassSchema();
        LOG.info("Wrapper schema " + wrapperSchema.toString());
        List<IndexedRecord> records = new ArrayList<>();
        for (HoodieInstant hoodieInstant : instants) {
            try {
                if (table.getActiveTimeline().isEmpty(hoodieInstant) && (hoodieInstant.getAction().equals(HoodieTimeline.CLEAN_ACTION) || (hoodieInstant.getAction().equals(HoodieTimeline.ROLLBACK_ACTION) && hoodieInstant.isCompleted()))) {
                    table.getActiveTimeline().deleteEmptyInstantIfExists(hoodieInstant);
                } else {
                    deleteAnyLeftOverMarkers(context, hoodieInstant);
                    records.add(convertToAvroRecord(hoodieInstant));
                    if (records.size() >= this.config.getCommitArchivalBatchSize()) {
                        writeToFile(wrapperSchema, records);
                    }
                }
            } catch (Exception e) {
                LOG.error("Failed to archive commits, .commit file: " + hoodieInstant.getFileName(), e);
                if (this.config.isFailOnTimelineArchivingEnabled()) {
                    throw e;
                }
            }
        }
        writeToFile(wrapperSchema, records);
    } catch (Exception e) {
        throw new HoodieCommitException("Failed to archive commits", e);
    }
}

Example 14 with HoodieCommitException

use of org.apache.hudi.exception.HoodieCommitException in project hudi by apache.

the class BaseJavaCommitActionExecutor method commit.

protected void commit(Option<Map<String, String>> extraMetadata, HoodieWriteMetadata<List<WriteStatus>> result, List<HoodieWriteStat> writeStats) {
    String actionType = getCommitActionType();
    LOG.info("Committing " + instantTime + ", action Type " + actionType);
    result.setCommitted(true);
    result.setWriteStats(writeStats);
    // Finalize write
    finalizeWrite(instantTime, writeStats, result);
    try {
        LOG.info("Committing " + instantTime + ", action Type " + getCommitActionType());
        HoodieActiveTimeline activeTimeline = table.getActiveTimeline();
        HoodieCommitMetadata metadata = result.getCommitMetadata().get();
        writeTableMetadata(metadata, actionType);
        activeTimeline.saveAsComplete(new HoodieInstant(true, getCommitActionType(), instantTime), Option.of(metadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
        LOG.info("Committed " + instantTime);
        result.setCommitMetadata(Option.of(metadata));
    } catch (IOException e) {
        throw new HoodieCommitException("Failed to complete commit " + config.getBasePath() + " at time " + instantTime, e);
    }
}

Example 15 with HoodieCommitException

use of org.apache.hudi.exception.HoodieCommitException in project hudi by apache.

the class SparkRDDWriteClient method completeCompaction.

@Override
protected void completeCompaction(HoodieCommitMetadata metadata, HoodieTable table, String compactionCommitTime) {
    this.context.setJobStatus(this.getClass().getSimpleName(), "Collect compaction write status and commit compaction");
    List<HoodieWriteStat> writeStats = metadata.getWriteStats();
    final HoodieInstant compactionInstant = new HoodieInstant(HoodieInstant.State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, compactionCommitTime);
    try {
        this.txnManager.beginTransaction(Option.of(compactionInstant), Option.empty());
        finalizeWrite(table, compactionCommitTime, writeStats);
        // commit to data table after committing to metadata table.
        updateTableMetadata(table, metadata, compactionInstant);
        LOG.info("Committing Compaction " + compactionCommitTime + ". Finished with result " + metadata);
        CompactHelpers.getInstance().completeInflightCompaction(table, compactionCommitTime, metadata);
    } finally {
        this.txnManager.endTransaction(Option.of(compactionInstant));
    }
    WriteMarkersFactory.get(config.getMarkersType(), table, compactionCommitTime).quietDeleteMarkerDir(context, config.getMarkersDeleteParallelism());
    if (compactionTimer != null) {
        long durationInMs = metrics.getDurationInMs(compactionTimer.stop());
        try {
            metrics.updateCommitMetrics(HoodieActiveTimeline.parseDateFromInstantTime(compactionCommitTime).getTime(), durationInMs, metadata, HoodieActiveTimeline.COMPACTION_ACTION);
        } catch (ParseException e) {
            throw new HoodieCommitException("Commit time is not of valid format. Failed to commit compaction " + config.getBasePath() + " at time " + compactionCommitTime, e);
        }
    }
    LOG.info("Compacted successfully on commit " + compactionCommitTime);
}