Examples with HoodieMetadataFileInfo org.apache.hudi.avro.model.HoodieMetadataFileInfo used on opensource projects

Example 1 with HoodieMetadataFileInfo

use of org.apache.hudi.avro.model.HoodieMetadataFileInfo in project hudi by apache.

the class HoodieMetadataPayload method createPartitionFilesRecord.

/**
 * Create and return a {@code HoodieMetadataPayload} to save list of files within a partition.
 *
 * @param partition    The name of the partition
 * @param filesAdded   Mapping of files to their sizes for files which have been added to this partition
 * @param filesDeleted List of files which have been deleted from this partition
 */
public static HoodieRecord<HoodieMetadataPayload> createPartitionFilesRecord(String partition, Option<Map<String, Long>> filesAdded, Option<List<String>> filesDeleted) {
    Map<String, HoodieMetadataFileInfo> fileInfo = new HashMap<>();
    filesAdded.ifPresent(filesMap -> fileInfo.putAll(filesMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, (entry) -> {
        long fileSize = entry.getValue();
        // Assert that the file-size of the file being added is positive, since Hudi
        // should not be creating empty files
        checkState(fileSize > 0);
        return new HoodieMetadataFileInfo(fileSize, false);
    }))));
    filesDeleted.ifPresent(filesList -> fileInfo.putAll(filesList.stream().collect(Collectors.toMap(Function.identity(), (ignored) -> new HoodieMetadataFileInfo(0L, true)))));
    HoodieKey key = new HoodieKey(partition, MetadataPartitionType.FILES.getPartitionPath());
    HoodieMetadataPayload payload = new HoodieMetadataPayload(key.getRecordKey(), METADATA_TYPE_FILE_LIST, fileInfo);
    return new HoodieAvroRecord<>(key, payload);
}

Example 2 with HoodieMetadataFileInfo

use of org.apache.hudi.avro.model.HoodieMetadataFileInfo in project hudi by apache.

the class HoodieMetadataPayload method createPartitionListRecord.

/**
 * Create and return a {@code HoodieMetadataPayload} to save list of partitions.
 *
 * @param partitions The list of partitions
 */
public static HoodieRecord<HoodieMetadataPayload> createPartitionListRecord(List<String> partitions) {
    Map<String, HoodieMetadataFileInfo> fileInfo = new HashMap<>();
    partitions.forEach(partition -> fileInfo.put(partition, new HoodieMetadataFileInfo(0L, false)));
    HoodieKey key = new HoodieKey(RECORDKEY_PARTITION_LIST, MetadataPartitionType.FILES.getPartitionPath());
    HoodieMetadataPayload payload = new HoodieMetadataPayload(key.getRecordKey(), METADATA_TYPE_PARTITION_LIST, fileInfo);
    return new HoodieAvroRecord<>(key, payload);
}