Examples with HoodieRequestedReplaceMetadata org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata used on opensource projects

Example 11 with HoodieRequestedReplaceMetadata

use of org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata in project hudi by apache.

the class TestClusteringUtils method createRequestedReplaceInstantNotClustering.

private HoodieInstant createRequestedReplaceInstantNotClustering(String instantTime) throws IOException {
    HoodieInstant newRequestedInstant = new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.REPLACE_COMMIT_ACTION, instantTime);
    HoodieRequestedReplaceMetadata requestedReplaceMetadata = HoodieRequestedReplaceMetadata.newBuilder().setOperationType(WriteOperationType.UNKNOWN.name()).build();
    metaClient.getActiveTimeline().saveToPendingReplaceCommit(newRequestedInstant, TimelineMetadataUtils.serializeRequestedReplaceMetadata(requestedReplaceMetadata));
    return newRequestedInstant;
}

Example 12 with HoodieRequestedReplaceMetadata

use of org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata in project hudi by apache.

the class ClusteringPlanActionExecutor method execute.

@Override
public Option<HoodieClusteringPlan> execute() {
    Option<HoodieClusteringPlan> planOption = createClusteringPlan();
    if (planOption.isPresent()) {
        HoodieInstant clusteringInstant = new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.REPLACE_COMMIT_ACTION, instantTime);
        try {
            HoodieRequestedReplaceMetadata requestedReplaceMetadata = HoodieRequestedReplaceMetadata.newBuilder().setOperationType(WriteOperationType.CLUSTER.name()).setExtraMetadata(extraMetadata.orElse(Collections.emptyMap())).setClusteringPlan(planOption.get()).build();
            table.getActiveTimeline().saveToPendingReplaceCommit(clusteringInstant, TimelineMetadataUtils.serializeRequestedReplaceMetadata(requestedReplaceMetadata));
        } catch (IOException ioe) {
            throw new HoodieIOException("Exception scheduling clustering", ioe);
        }
    }
    return planOption;
}

Example 13 with HoodieRequestedReplaceMetadata

use of org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata in project hudi by apache.

the class MetadataConversionUtils method createMetaWrapper.

public static HoodieArchivedMetaEntry createMetaWrapper(HoodieInstant hoodieInstant, HoodieTableMetaClient metaClient) throws IOException {
    HoodieArchivedMetaEntry archivedMetaWrapper = new HoodieArchivedMetaEntry();
    archivedMetaWrapper.setCommitTime(hoodieInstant.getTimestamp());
    archivedMetaWrapper.setActionState(hoodieInstant.getState().name());
    switch(hoodieInstant.getAction()) {
        case HoodieTimeline.CLEAN_ACTION:
            {
                if (hoodieInstant.isCompleted()) {
                    archivedMetaWrapper.setHoodieCleanMetadata(CleanerUtils.getCleanerMetadata(metaClient, hoodieInstant));
                } else {
                    archivedMetaWrapper.setHoodieCleanerPlan(CleanerUtils.getCleanerPlan(metaClient, hoodieInstant));
                }
                archivedMetaWrapper.setActionType(ActionType.clean.name());
                break;
            }
        case HoodieTimeline.COMMIT_ACTION:
            {
                HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(metaClient.getActiveTimeline().getInstantDetails(hoodieInstant).get(), HoodieCommitMetadata.class);
                archivedMetaWrapper.setHoodieCommitMetadata(convertCommitMetadata(commitMetadata));
                archivedMetaWrapper.setActionType(ActionType.commit.name());
                break;
            }
        case HoodieTimeline.DELTA_COMMIT_ACTION:
            {
                HoodieCommitMetadata deltaCommitMetadata = HoodieCommitMetadata.fromBytes(metaClient.getActiveTimeline().getInstantDetails(hoodieInstant).get(), HoodieCommitMetadata.class);
                archivedMetaWrapper.setHoodieCommitMetadata(convertCommitMetadata(deltaCommitMetadata));
                archivedMetaWrapper.setActionType(ActionType.deltacommit.name());
                break;
            }
        case HoodieTimeline.REPLACE_COMMIT_ACTION:
            {
                if (hoodieInstant.isCompleted()) {
                    HoodieReplaceCommitMetadata replaceCommitMetadata = HoodieReplaceCommitMetadata.fromBytes(metaClient.getActiveTimeline().getInstantDetails(hoodieInstant).get(), HoodieReplaceCommitMetadata.class);
                    archivedMetaWrapper.setHoodieReplaceCommitMetadata(ReplaceArchivalHelper.convertReplaceCommitMetadata(replaceCommitMetadata));
                } else if (hoodieInstant.isInflight()) {
                    // inflight replacecommit files have the same meta data body as HoodieCommitMetadata
                    // so we could re-use it without further creating an inflight extension.
                    // Or inflight replacecommit files are empty under clustering circumstance
                    Option<HoodieCommitMetadata> inflightCommitMetadata = getInflightReplaceMetadata(metaClient, hoodieInstant);
                    if (inflightCommitMetadata.isPresent()) {
                        archivedMetaWrapper.setHoodieInflightReplaceMetadata(convertCommitMetadata(inflightCommitMetadata.get()));
                    }
                } else {
                    // we may have cases with empty HoodieRequestedReplaceMetadata e.g. insert_overwrite_table or insert_overwrite
                    // without clustering. However, we should revisit the requested commit file standardization
                    Option<HoodieRequestedReplaceMetadata> requestedReplaceMetadata = getRequestedReplaceMetadata(metaClient, hoodieInstant);
                    if (requestedReplaceMetadata.isPresent()) {
                        archivedMetaWrapper.setHoodieRequestedReplaceMetadata(requestedReplaceMetadata.get());
                    }
                }
                archivedMetaWrapper.setActionType(ActionType.replacecommit.name());
                break;
            }
        case HoodieTimeline.ROLLBACK_ACTION:
            {
                if (hoodieInstant.isCompleted()) {
                    archivedMetaWrapper.setHoodieRollbackMetadata(TimelineMetadataUtils.deserializeAvroMetadata(metaClient.getActiveTimeline().getInstantDetails(hoodieInstant).get(), HoodieRollbackMetadata.class));
                }
                archivedMetaWrapper.setActionType(ActionType.rollback.name());
                break;
            }
        case HoodieTimeline.SAVEPOINT_ACTION:
            {
                archivedMetaWrapper.setHoodieSavePointMetadata(TimelineMetadataUtils.deserializeAvroMetadata(metaClient.getActiveTimeline().getInstantDetails(hoodieInstant).get(), HoodieSavepointMetadata.class));
                archivedMetaWrapper.setActionType(ActionType.savepoint.name());
                break;
            }
        case HoodieTimeline.COMPACTION_ACTION:
            {
                HoodieCompactionPlan plan = CompactionUtils.getCompactionPlan(metaClient, hoodieInstant.getTimestamp());
                archivedMetaWrapper.setHoodieCompactionPlan(plan);
                archivedMetaWrapper.setActionType(ActionType.compaction.name());
                break;
            }
        default:
            {
                throw new UnsupportedOperationException("Action not fully supported yet");
            }
    }
    return archivedMetaWrapper;
}

Example 14 with HoodieRequestedReplaceMetadata

use of org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata in project hudi by apache.

the class ConcurrentOperation method init.

private void init(HoodieInstant instant) {
    if (this.metadataWrapper.isAvroMetadata()) {
        switch(getInstantActionType()) {
            case COMPACTION_ACTION:
                this.operationType = WriteOperationType.COMPACT;
                this.mutatedFileIds = this.metadataWrapper.getMetadataFromTimeline().getHoodieCompactionPlan().getOperations().stream().map(op -> op.getFileId()).collect(Collectors.toSet());
                break;
            case COMMIT_ACTION:
            case DELTA_COMMIT_ACTION:
                this.mutatedFileIds = CommitUtils.getFileIdWithoutSuffixAndRelativePathsFromSpecificRecord(this.metadataWrapper.getMetadataFromTimeline().getHoodieCommitMetadata().getPartitionToWriteStats()).keySet();
                this.operationType = WriteOperationType.fromValue(this.metadataWrapper.getMetadataFromTimeline().getHoodieCommitMetadata().getOperationType());
                break;
            case REPLACE_COMMIT_ACTION:
                if (instant.isCompleted()) {
                    this.mutatedFileIds = CommitUtils.getFileIdWithoutSuffixAndRelativePathsFromSpecificRecord(this.metadataWrapper.getMetadataFromTimeline().getHoodieReplaceCommitMetadata().getPartitionToWriteStats()).keySet();
                    this.operationType = WriteOperationType.fromValue(this.metadataWrapper.getMetadataFromTimeline().getHoodieReplaceCommitMetadata().getOperationType());
                } else {
                    HoodieRequestedReplaceMetadata requestedReplaceMetadata = this.metadataWrapper.getMetadataFromTimeline().getHoodieRequestedReplaceMetadata();
                    this.mutatedFileIds = requestedReplaceMetadata.getClusteringPlan().getInputGroups().stream().flatMap(ig -> ig.getSlices().stream()).map(file -> file.getFileId()).collect(Collectors.toSet());
                    this.operationType = WriteOperationType.CLUSTER;
                }
                break;
            default:
                throw new IllegalArgumentException("Unsupported Action Type " + getInstantActionType());
        }
    } else {
        switch(getInstantActionType()) {
            case COMMIT_ACTION:
            case DELTA_COMMIT_ACTION:
                this.mutatedFileIds = CommitUtils.getFileIdWithoutSuffixAndRelativePaths(this.metadataWrapper.getCommitMetadata().getPartitionToWriteStats()).keySet();
                this.operationType = this.metadataWrapper.getCommitMetadata().getOperationType();
                break;
            default:
                throw new IllegalArgumentException("Unsupported Action Type " + getInstantActionType());
        }
    }
}

Example 15 with HoodieRequestedReplaceMetadata

use of org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata in project hudi by apache.

the class TestHoodieClientOnCopyOnWriteStorage method createRequestedReplaceInstant.

protected HoodieInstant createRequestedReplaceInstant(HoodieTableMetaClient metaClient, String clusterTime, List<FileSlice>[] fileSlices) throws IOException {
    HoodieClusteringPlan clusteringPlan = ClusteringUtils.createClusteringPlan(EXECUTION_STRATEGY_CLASS_NAME.defaultValue(), STRATEGY_PARAMS, fileSlices, Collections.emptyMap());
    HoodieInstant clusteringInstant = new HoodieInstant(REQUESTED, REPLACE_COMMIT_ACTION, clusterTime);
    HoodieRequestedReplaceMetadata requestedReplaceMetadata = HoodieRequestedReplaceMetadata.newBuilder().setClusteringPlan(clusteringPlan).setOperationType(WriteOperationType.CLUSTER.name()).build();
    metaClient.getActiveTimeline().saveToPendingReplaceCommit(clusteringInstant, TimelineMetadataUtils.serializeRequestedReplaceMetadata(requestedReplaceMetadata));
    return clusteringInstant;
}