Examples with HoodieCleanPartitionMetadata org.apache.hudi.avro.model.HoodieCleanPartitionMetadata used on opensource projects

Example 6 with HoodieCleanPartitionMetadata

use of org.apache.hudi.avro.model.HoodieCleanPartitionMetadata in project hudi by apache.

the class TestCleaner method runCleaner.

/**
 * Helper to run cleaner and collect Clean Stats.
 *
 * @param config HoodieWriteConfig
 */
private List<HoodieCleanStat> runCleaner(HoodieWriteConfig config, boolean simulateRetryFailure, int firstCommitSequence) throws IOException {
    SparkRDDWriteClient<?> writeClient = getHoodieWriteClient(config);
    String cleanInstantTs = makeNewCommitTime(firstCommitSequence);
    HoodieCleanMetadata cleanMetadata1 = writeClient.clean(cleanInstantTs);
    if (null == cleanMetadata1) {
        return new ArrayList<>();
    }
    if (simulateRetryFailure) {
        HoodieInstant completedCleanInstant = new HoodieInstant(State.COMPLETED, HoodieTimeline.CLEAN_ACTION, cleanInstantTs);
        HoodieCleanMetadata metadata = CleanerUtils.getCleanerMetadata(metaClient, completedCleanInstant);
        metadata.getPartitionMetadata().values().forEach(p -> {
            String dirPath = metaClient.getBasePath() + "/" + p.getPartitionPath();
            p.getSuccessDeleteFiles().forEach(p2 -> {
                try {
                    metaClient.getFs().create(new Path(dirPath, p2), true);
                } catch (IOException e) {
                    throw new HoodieIOException(e.getMessage(), e);
                }
            });
        });
        metaClient.reloadActiveTimeline().revertToInflight(completedCleanInstant);
        // retry clean operation again
        writeClient.clean();
        final HoodieCleanMetadata retriedCleanMetadata = CleanerUtils.getCleanerMetadata(HoodieTableMetaClient.reload(metaClient), completedCleanInstant);
        cleanMetadata1.getPartitionMetadata().keySet().forEach(k -> {
            HoodieCleanPartitionMetadata p1 = cleanMetadata1.getPartitionMetadata().get(k);
            HoodieCleanPartitionMetadata p2 = retriedCleanMetadata.getPartitionMetadata().get(k);
            assertEquals(p1.getDeletePathPatterns(), p2.getDeletePathPatterns());
            assertEquals(p1.getSuccessDeleteFiles(), p2.getSuccessDeleteFiles());
            assertEquals(p1.getFailedDeleteFiles(), p2.getFailedDeleteFiles());
            assertEquals(p1.getPartitionPath(), p2.getPartitionPath());
            assertEquals(k, p1.getPartitionPath());
        });
    }
    Map<String, HoodieCleanStat> cleanStatMap = cleanMetadata1.getPartitionMetadata().values().stream().map(x -> new HoodieCleanStat.Builder().withPartitionPath(x.getPartitionPath()).withFailedDeletes(x.getFailedDeleteFiles()).withSuccessfulDeletes(x.getSuccessDeleteFiles()).withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy())).withDeletePathPattern(x.getDeletePathPatterns()).withEarliestCommitRetained(Option.ofNullable(cleanMetadata1.getEarliestCommitToRetain() != null ? new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "000") : null)).build()).collect(Collectors.toMap(HoodieCleanStat::getPartitionPath, x -> x));
    cleanMetadata1.getBootstrapPartitionMetadata().values().forEach(x -> {
        HoodieCleanStat s = cleanStatMap.get(x.getPartitionPath());
        cleanStatMap.put(x.getPartitionPath(), new HoodieCleanStat.Builder().withPartitionPath(x.getPartitionPath()).withFailedDeletes(s.getFailedDeleteFiles()).withSuccessfulDeletes(s.getSuccessDeleteFiles()).withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy())).withDeletePathPattern(s.getDeletePathPatterns()).withEarliestCommitRetained(Option.ofNullable(s.getEarliestCommitToRetain()).map(y -> new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, y))).withSuccessfulDeleteBootstrapBaseFiles(x.getSuccessDeleteFiles()).withFailedDeleteBootstrapBaseFiles(x.getFailedDeleteFiles()).withDeleteBootstrapBasePathPatterns(x.getDeletePathPatterns()).build());
    });
    return new ArrayList<>(cleanStatMap.values());
}

Example 7 with HoodieCleanPartitionMetadata

use of org.apache.hudi.avro.model.HoodieCleanPartitionMetadata in project hudi by apache.

the class TestTimelineUtils method getCleanMetadata.

private Option<byte[]> getCleanMetadata(String partition, String time) throws IOException {
    Map<String, HoodieCleanPartitionMetadata> partitionToFilesCleaned = new HashMap<>();
    List<String> filesDeleted = new ArrayList<>();
    filesDeleted.add("file-" + partition + "-" + time + "1");
    filesDeleted.add("file-" + partition + "-" + time + "2");
    HoodieCleanPartitionMetadata partitionMetadata = HoodieCleanPartitionMetadata.newBuilder().setPartitionPath(partition).setPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS.name()).setFailedDeleteFiles(Collections.emptyList()).setDeletePathPatterns(Collections.emptyList()).setSuccessDeleteFiles(filesDeleted).build();
    partitionToFilesCleaned.putIfAbsent(partition, partitionMetadata);
    HoodieCleanMetadata cleanMetadata = HoodieCleanMetadata.newBuilder().setVersion(1).setTimeTakenInMillis(100).setTotalFilesDeleted(1).setStartCleanTime(time).setEarliestCommitToRetain(time).setPartitionMetadata(partitionToFilesCleaned).build();
    return TimelineMetadataUtils.serializeCleanMetadata(cleanMetadata);
}