Examples with HoodieCommitMetadata org.apache.hudi.common.model.HoodieCommitMetadata used on opensource projects

Example 86 with HoodieCommitMetadata

use of org.apache.hudi.common.model.HoodieCommitMetadata in project hudi by apache.

the class HoodieTestTable method doRollbackWithExtraFiles.

public HoodieTestTable doRollbackWithExtraFiles(String commitTimeToRollback, String commitTime, Map<String, List<String>> extraFiles) throws Exception {
    metaClient = HoodieTableMetaClient.reload(metaClient);
    Option<HoodieCommitMetadata> commitMetadata = getMetadataForInstant(commitTimeToRollback);
    if (!commitMetadata.isPresent()) {
        throw new IllegalArgumentException("Instant to rollback not present in timeline: " + commitTimeToRollback);
    }
    Map<String, List<String>> partitionFiles = getPartitionFiles(commitMetadata.get());
    for (Map.Entry<String, List<String>> entry : partitionFiles.entrySet()) {
        deleteFilesInPartition(entry.getKey(), entry.getValue());
    }
    for (Map.Entry<String, List<String>> entry : extraFiles.entrySet()) {
        if (partitionFiles.containsKey(entry.getKey())) {
            partitionFiles.get(entry.getKey()).addAll(entry.getValue());
        }
    }
    HoodieRollbackMetadata rollbackMetadata = getRollbackMetadata(commitTimeToRollback, partitionFiles);
    return addRollback(commitTime, rollbackMetadata);
}

Example 87 with HoodieCommitMetadata

use of org.apache.hudi.common.model.HoodieCommitMetadata in project hudi by apache.

the class HoodieTestTable method getMetadataForInstant.

private Option<HoodieCommitMetadata> getMetadataForInstant(String instantTime) {
    metaClient = HoodieTableMetaClient.reload(metaClient);
    Option<HoodieInstant> hoodieInstant = metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().filter(i -> i.getTimestamp().equals(instantTime)).firstInstant();
    try {
        if (hoodieInstant.isPresent()) {
            return getCommitMeta(hoodieInstant.get());
        } else {
            return Option.empty();
        }
    } catch (IOException io) {
        throw new HoodieIOException("Unable to read metadata for instant " + hoodieInstant.get(), io);
    }
}

Example 88 with HoodieCommitMetadata

use of org.apache.hudi.common.model.HoodieCommitMetadata in project hudi by apache.

the class HoodieTestTable method doSavepoint.

public HoodieSavepointMetadata doSavepoint(String commitTime) throws IOException {
    Option<HoodieCommitMetadata> commitMetadata = getMetadataForInstant(commitTime);
    if (!commitMetadata.isPresent()) {
        throw new IllegalArgumentException("Instant to rollback not present in timeline: " + commitTime);
    }
    Map<String, List<String>> partitionFiles = getPartitionFiles(commitMetadata.get());
    HoodieSavepointMetadata savepointMetadata = getSavepointMetadata(commitTime, partitionFiles);
    for (Map.Entry<String, List<String>> entry : partitionFiles.entrySet()) {
        deleteFilesInPartition(entry.getKey(), entry.getValue());
    }
    return savepointMetadata;
}

Example 89 with HoodieCommitMetadata

use of org.apache.hudi.common.model.HoodieCommitMetadata in project hudi by apache.

the class HoodieTestTable method doCompaction.

public HoodieCommitMetadata doCompaction(String commitTime, List<String> partitions, boolean inflight) throws Exception {
    this.currentInstantTime = commitTime;
    if (partitions.isEmpty()) {
        partitions = Collections.singletonList(EMPTY_STRING);
    }
    HoodieTestTableState testTableState = getTestTableStateWithPartitionFileInfo(COMPACT, metaClient.getTableType(), commitTime, partitions, 1);
    HoodieCommitMetadata commitMetadata = createCommitMetadata(COMPACT, commitTime, testTableState);
    for (String partition : partitions) {
        this.withBaseFilesInPartition(partition, testTableState.getPartitionToBaseFileInfoMap(commitTime).get(partition));
    }
    if (inflight) {
        this.addInflightCompaction(commitTime, commitMetadata);
    } else {
        this.addCompaction(commitTime, commitMetadata);
    }
    return commitMetadata;
}

Example 90 with HoodieCommitMetadata

use of org.apache.hudi.common.model.HoodieCommitMetadata in project hudi by apache.

the class HoodieTestTable method doRestore.

public HoodieTestTable doRestore(String commitToRestoreTo, String restoreTime) throws Exception {
    metaClient = HoodieTableMetaClient.reload(metaClient);
    List<HoodieInstant> commitsToRollback = metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().findInstantsAfter(commitToRestoreTo).getReverseOrderedInstants().collect(Collectors.toList());
    Map<String, List<HoodieRollbackMetadata>> rollbackMetadataMap = new HashMap<>();
    for (HoodieInstant commitInstantToRollback : commitsToRollback) {
        Option<HoodieCommitMetadata> commitMetadata = getCommitMeta(commitInstantToRollback);
        if (!commitMetadata.isPresent()) {
            throw new IllegalArgumentException("Instant to rollback not present in timeline: " + commitInstantToRollback.getTimestamp());
        }
        Map<String, List<String>> partitionFiles = getPartitionFiles(commitMetadata.get());
        rollbackMetadataMap.put(commitInstantToRollback.getTimestamp(), Collections.singletonList(getRollbackMetadata(commitInstantToRollback.getTimestamp(), partitionFiles)));
        for (Map.Entry<String, List<String>> entry : partitionFiles.entrySet()) {
            deleteFilesInPartition(entry.getKey(), entry.getValue());
        }
    }
    HoodieRestoreMetadata restoreMetadata = TimelineMetadataUtils.convertRestoreMetadata(restoreTime, 1000L, commitsToRollback, rollbackMetadataMap);
    return addRestore(restoreTime, restoreMetadata);
}