Examples with HoodieCommitMetadata org.apache.hudi.common.model.HoodieCommitMetadata used on opensource projects

Example 41 with HoodieCommitMetadata

use of org.apache.hudi.common.model.HoodieCommitMetadata in project hudi by apache.

the class TestTimelineUtils method getCommitMetadata.

private byte[] getCommitMetadata(String basePath, String partition, String commitTs, int count, Map<String, String> extraMetadata) throws IOException {
    HoodieCommitMetadata commit = new HoodieCommitMetadata();
    for (int i = 1; i <= count; i++) {
        HoodieWriteStat stat = new HoodieWriteStat();
        stat.setFileId(i + "");
        stat.setPartitionPath(Paths.get(basePath, partition).toString());
        stat.setPath(commitTs + "." + i + metaClient.getTableConfig().getBaseFileFormat().getFileExtension());
        commit.addWriteStat(partition, stat);
    }
    for (Map.Entry<String, String> extraEntries : extraMetadata.entrySet()) {
        commit.addMetadata(extraEntries.getKey(), extraEntries.getValue());
    }
    return commit.toJsonString().getBytes(StandardCharsets.UTF_8);
}

Example 42 with HoodieCommitMetadata

use of org.apache.hudi.common.model.HoodieCommitMetadata in project hudi by apache.

the class HoodieTestDataGenerator method createSavepointFile.

public static void createSavepointFile(String basePath, String instantTime, Configuration configuration) throws IOException {
    Path commitFile = new Path(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTimeline.makeSavePointFileName(instantTime));
    FileSystem fs = FSUtils.getFs(basePath, configuration);
    try (FSDataOutputStream os = fs.create(commitFile, true)) {
        HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
        // Write empty commit metadata
        os.writeBytes(new String(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
    }
}

Example 43 with HoodieCommitMetadata

use of org.apache.hudi.common.model.HoodieCommitMetadata in project hudi by apache.

the class HoodieTestDataGenerator method createCommitFile.

public static void createCommitFile(String basePath, String instantTime, Configuration configuration) {
    HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
    createCommitFile(basePath, instantTime, configuration, commitMetadata);
}

Example 44 with HoodieCommitMetadata

use of org.apache.hudi.common.model.HoodieCommitMetadata in project hudi by apache.

the class HoodieTestTable method doWriteOperation.

/**
 * Add commits to the requested partitions.
 *
 * @param commitTime                    - Commit time for the operation
 * @param operationType                 - Operation type
 * @param newPartitionsToAdd            - New partitions to add for the operation
 * @param partitionToFilesNameLengthMap - Map of partition names to its list of files name and length pair
 * @param bootstrap                     - Whether bootstrapping needed for the operation
 * @param createInflightCommit          - Whether in flight commit needed for the operation
 * @return Commit metadata for the commit operation performed.
 * @throws Exception
 */
public HoodieCommitMetadata doWriteOperation(String commitTime, WriteOperationType operationType, List<String> newPartitionsToAdd, Map<String, List<Pair<String, Integer>>> partitionToFilesNameLengthMap, boolean bootstrap, boolean createInflightCommit) throws Exception {
    if (partitionToFilesNameLengthMap.isEmpty()) {
        partitionToFilesNameLengthMap = Collections.singletonMap(EMPTY_STRING, Collections.EMPTY_LIST);
    }
    HoodieTestTableState testTableState = getTestTableStateWithPartitionFileInfo(operationType, metaClient.getTableType(), commitTime, partitionToFilesNameLengthMap);
    HoodieCommitMetadata commitMetadata = createCommitMetadata(operationType, commitTime, testTableState, bootstrap);
    for (String str : newPartitionsToAdd) {
        this.withPartitionMetaFiles(str);
    }
    if (createInflightCommit) {
        if (metaClient.getTableType() == HoodieTableType.COPY_ON_WRITE) {
            this.addInflightCommit(commitTime);
        } else {
            this.addInflightDeltaCommit(commitTime);
        }
    } else {
        if (metaClient.getTableType() == HoodieTableType.COPY_ON_WRITE) {
            this.addCommit(commitTime, Option.of(commitMetadata));
        } else {
            this.addDeltaCommit(commitTime, commitMetadata);
        }
    }
    for (Map.Entry<String, List<Pair<String, Integer>>> entry : partitionToFilesNameLengthMap.entrySet()) {
        String partition = entry.getKey();
        this.withBaseFilesInPartition(partition, testTableState.getPartitionToBaseFileInfoMap(commitTime).get(partition));
        if (MERGE_ON_READ.equals(metaClient.getTableType()) && UPSERT.equals(operationType)) {
            this.withLogFilesInPartition(partition, testTableState.getPartitionToLogFileInfoMap(commitTime).get(partition));
        }
    }
    return commitMetadata;
}

Example 45 with HoodieCommitMetadata

use of org.apache.hudi.common.model.HoodieCommitMetadata in project hudi by apache.

the class HoodieTestTable method doRollback.

public HoodieTestTable doRollback(String commitTimeToRollback, String commitTime) throws Exception {
    metaClient = HoodieTableMetaClient.reload(metaClient);
    Option<HoodieCommitMetadata> commitMetadata = getMetadataForInstant(commitTimeToRollback);
    if (!commitMetadata.isPresent()) {
        throw new IllegalArgumentException("Instant to rollback not present in timeline: " + commitTimeToRollback);
    }
    Map<String, List<String>> partitionFiles = getPartitionFiles(commitMetadata.get());
    HoodieRollbackMetadata rollbackMetadata = getRollbackMetadata(commitTimeToRollback, partitionFiles);
    for (Map.Entry<String, List<String>> entry : partitionFiles.entrySet()) {
        deleteFilesInPartition(entry.getKey(), entry.getValue());
    }
    return addRollback(commitTime, rollbackMetadata);
}