Examples with MiniBatchHandle org.apache.hudi.io.MiniBatchHandle used on opensource projects

Example 1 with MiniBatchHandle

use of org.apache.hudi.io.MiniBatchHandle in project hudi by apache.

the class HoodieFlinkWriteClient method getOrCreateWriteHandle.

/**
 * Get or create a new write handle in order to reuse the file handles.
 *
 * @param record      The first record in the bucket
 * @param config      Write config
 * @param instantTime The instant time
 * @param table       The table
 * @param recordItr   Record iterator
 * @return Existing write handle or create a new one
 */
private HoodieWriteHandle<?, ?, ?, ?> getOrCreateWriteHandle(HoodieRecord<T> record, HoodieWriteConfig config, String instantTime, HoodieTable<T, List<HoodieRecord<T>>, List<HoodieKey>, List<WriteStatus>> table, Iterator<HoodieRecord<T>> recordItr) {
    final HoodieRecordLocation loc = record.getCurrentLocation();
    final String fileID = loc.getFileId();
    final String partitionPath = record.getPartitionPath();
    final boolean insertClustering = config.allowDuplicateInserts();
    if (bucketToHandles.containsKey(fileID)) {
        MiniBatchHandle lastHandle = (MiniBatchHandle) bucketToHandles.get(fileID);
        if (lastHandle.shouldReplace()) {
            HoodieWriteHandle<?, ?, ?, ?> writeHandle = insertClustering ? new FlinkConcatAndReplaceHandle<>(config, instantTime, table, recordItr, partitionPath, fileID, table.getTaskContextSupplier(), lastHandle.getWritePath()) : new FlinkMergeAndReplaceHandle<>(config, instantTime, table, recordItr, partitionPath, fileID, table.getTaskContextSupplier(), lastHandle.getWritePath());
            // override with new replace handle
            this.bucketToHandles.put(fileID, writeHandle);
            return writeHandle;
        }
    }
    final boolean isDelta = table.getMetaClient().getTableType().equals(HoodieTableType.MERGE_ON_READ);
    final HoodieWriteHandle<?, ?, ?, ?> writeHandle;
    if (isDelta) {
        writeHandle = new FlinkAppendHandle<>(config, instantTime, table, partitionPath, fileID, recordItr, table.getTaskContextSupplier());
    } else if (loc.getInstantTime().equals("I")) {
        writeHandle = new FlinkCreateHandle<>(config, instantTime, table, partitionPath, fileID, table.getTaskContextSupplier());
    } else {
        writeHandle = insertClustering ? new FlinkConcatHandle<>(config, instantTime, table, recordItr, partitionPath, fileID, table.getTaskContextSupplier()) : new FlinkMergeHandle<>(config, instantTime, table, recordItr, partitionPath, fileID, table.getTaskContextSupplier());
    }
    this.bucketToHandles.put(fileID, writeHandle);
    return writeHandle;
}