Examples with HoodieWriteStat org.apache.hudi.common.model.HoodieWriteStat used on opensource projects

Example 41 with HoodieWriteStat

use of org.apache.hudi.common.model.HoodieWriteStat in project hudi by apache.

the class HoodieTestCommitGenerator method generateCommitMetadata.

public static HoodieCommitMetadata generateCommitMetadata(Map<String, List<Pair<String, String>>> partitionPathToFileIdAndNameMap, Map<String, String> extraMetadata) {
    HoodieCommitMetadata metadata = new HoodieCommitMetadata();
    for (Map.Entry<String, String> entry : extraMetadata.entrySet()) {
        metadata.addMetadata(entry.getKey(), entry.getValue());
    }
    partitionPathToFileIdAndNameMap.forEach((partitionPath, fileInfoList) -> fileInfoList.forEach(fileInfo -> {
        HoodieWriteStat writeStat = new HoodieWriteStat();
        writeStat.setPartitionPath(partitionPath);
        writeStat.setPath(new Path(partitionPath, fileInfo.getValue()).toString());
        writeStat.setFileId(fileInfo.getKey());
        // Below are dummy values
        writeStat.setTotalWriteBytes(10000);
        writeStat.setPrevCommit("000");
        writeStat.setNumWrites(10);
        writeStat.setNumUpdateWrites(15);
        writeStat.setTotalLogBlocks(2);
        writeStat.setTotalLogRecords(100);
        metadata.addWriteStat(partitionPath, writeStat);
    }));
    return metadata;
}

Example 42 with HoodieWriteStat

use of org.apache.hudi.common.model.HoodieWriteStat in project hudi by apache.

the class TestSimpleConcurrentFileWritesConflictResolutionStrategy method createCommitMetadata.

private HoodieCommitMetadata createCommitMetadata(String instantTime) {
    HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
    commitMetadata.addMetadata("test", "test");
    HoodieWriteStat writeStat = new HoodieWriteStat();
    writeStat.setFileId("file-1");
    commitMetadata.addWriteStat(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, writeStat);
    commitMetadata.setOperationType(WriteOperationType.INSERT);
    return commitMetadata;
}

Example 43 with HoodieWriteStat

use of org.apache.hudi.common.model.HoodieWriteStat in project hudi by apache.

the class TestSimpleConcurrentFileWritesConflictResolutionStrategy method createCompaction.

private void createCompaction(String instantTime) throws Exception {
    String fileId1 = "file-1";
    String fileId2 = "file-2";
    HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
    commitMetadata.addMetadata("test", "test");
    commitMetadata.setOperationType(WriteOperationType.COMPACT);
    commitMetadata.setCompacted(true);
    HoodieWriteStat writeStat = new HoodieWriteStat();
    writeStat.setFileId("file-1");
    commitMetadata.addWriteStat(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, writeStat);
    HoodieTestTable.of(metaClient).addCommit(instantTime, Option.of(commitMetadata)).withBaseFilesInPartition(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, fileId1, fileId2);
}

Example 44 with HoodieWriteStat

use of org.apache.hudi.common.model.HoodieWriteStat in project hudi by apache.

the class TestSimpleConcurrentFileWritesConflictResolutionStrategy method createCommit.

private void createCommit(String instantTime) throws Exception {
    String fileId1 = "file-1";
    String fileId2 = "file-2";
    HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
    commitMetadata.addMetadata("test", "test");
    HoodieWriteStat writeStat = new HoodieWriteStat();
    writeStat.setFileId("file-1");
    commitMetadata.addWriteStat(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, writeStat);
    commitMetadata.setOperationType(WriteOperationType.INSERT);
    HoodieTestTable.of(metaClient).addCommit(instantTime, Option.of(commitMetadata)).withBaseFilesInPartition(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, fileId1, fileId2);
}

Example 45 with HoodieWriteStat

use of org.apache.hudi.common.model.HoodieWriteStat in project hudi by apache.

the class HoodieTestTable method addInflightCompaction.

public HoodieTestTable addInflightCompaction(String instantTime, HoodieCommitMetadata commitMetadata) throws Exception {
    List<FileSlice> fileSlices = new ArrayList<>();
    for (Map.Entry<String, List<HoodieWriteStat>> entry : commitMetadata.getPartitionToWriteStats().entrySet()) {
        for (HoodieWriteStat stat : entry.getValue()) {
            fileSlices.add(new FileSlice(entry.getKey(), instantTime, stat.getPath()));
        }
    }
    this.addRequestedCompaction(instantTime, fileSlices.toArray(new FileSlice[0]));
    createInflightCompaction(basePath, instantTime);
    inflightCommits.add(instantTime);
    currentInstantTime = instantTime;
    return this;
}