Examples with HoodieAppendHandle org.apache.hudi.io.HoodieAppendHandle used on opensource projects

Example 1 with HoodieAppendHandle

use of org.apache.hudi.io.HoodieAppendHandle in project hudi by apache.

the class JavaUpsertPreppedDeltaCommitActionExecutor method execute.

@Override
public HoodieWriteMetadata<List<WriteStatus>> execute() {
    HoodieWriteMetadata<List<WriteStatus>> result = new HoodieWriteMetadata<>();
    // First group by target file id.
    HashMap<Pair<String, String>, List<HoodieRecord<T>>> recordsByFileId = new HashMap<>();
    List<HoodieRecord<T>> insertedRecords = new LinkedList<>();
    // Split records into inserts and updates.
    for (HoodieRecord<T> record : preppedInputRecords) {
        if (!record.isCurrentLocationKnown()) {
            insertedRecords.add(record);
        } else {
            Pair<String, String> fileIdPartitionPath = Pair.of(record.getCurrentLocation().getFileId(), record.getPartitionPath());
            if (!recordsByFileId.containsKey(fileIdPartitionPath)) {
                recordsByFileId.put(fileIdPartitionPath, new LinkedList<>());
            }
            recordsByFileId.get(fileIdPartitionPath).add(record);
        }
    }
    LOG.info(String.format("Total update fileIDs %s, total inserts %s for commit %s", recordsByFileId.size(), insertedRecords.size(), instantTime));
    List<WriteStatus> allWriteStatuses = new ArrayList<>();
    try {
        recordsByFileId.forEach((k, v) -> {
            HoodieAppendHandle<?, ?, ?, ?> appendHandle = new HoodieAppendHandle(config, instantTime, table, k.getRight(), k.getLeft(), v.iterator(), taskContextSupplier);
            appendHandle.doAppend();
            allWriteStatuses.addAll(appendHandle.close());
        });
        if (insertedRecords.size() > 0) {
            HoodieWriteMetadata<List<WriteStatus>> insertResult = JavaBulkInsertHelper.newInstance().bulkInsert(insertedRecords, instantTime, table, config, this, false, Option.empty());
            allWriteStatuses.addAll(insertResult.getWriteStatuses());
        }
    } catch (Throwable e) {
        if (e instanceof HoodieUpsertException) {
            throw e;
        }
        throw new HoodieUpsertException("Failed to upsert for commit time " + instantTime, e);
    }
    updateIndex(allWriteStatuses, result);
    return result;
}