Examples with HoodieArchivedMetaEntry org.apache.hudi.avro.model.HoodieArchivedMetaEntry used on opensource projects

Example 6 with HoodieArchivedMetaEntry

use of org.apache.hudi.avro.model.HoodieArchivedMetaEntry in project hudi by apache.

the class TestMetadataConversionUtils method testEmptyInflightReplace.

@Test
public void testEmptyInflightReplace() throws Exception {
    String newCommitTime = HoodieTestTable.makeNewCommitTime();
    createReplace(newCommitTime, WriteOperationType.INSERT_OVERWRITE_TABLE, true);
    HoodieArchivedMetaEntry metaEntry = MetadataConversionUtils.createMetaWrapper(new HoodieInstant(State.INFLIGHT, HoodieTimeline.REPLACE_COMMIT_ACTION, newCommitTime), metaClient);
    assertEquals(metaEntry.getActionState(), State.INFLIGHT.toString());
    assertNull(metaEntry.getHoodieInflightReplaceMetadata());
}

Example 7 with HoodieArchivedMetaEntry

use of org.apache.hudi.avro.model.HoodieArchivedMetaEntry in project hudi by apache.

the class TestMetadataConversionUtils method testEmptyRequestedReplace.

@Test
public void testEmptyRequestedReplace() throws Exception {
    String newCommitTime = HoodieTestTable.makeNewCommitTime();
    createReplace(newCommitTime, WriteOperationType.INSERT_OVERWRITE_TABLE, false);
    HoodieArchivedMetaEntry metaEntry = MetadataConversionUtils.createMetaWrapper(new HoodieInstant(State.REQUESTED, HoodieTimeline.REPLACE_COMMIT_ACTION, newCommitTime), metaClient);
    assertEquals(metaEntry.getActionState(), State.REQUESTED.toString());
    assertNull(metaEntry.getHoodieRequestedReplaceMetadata());
}

Example 8 with HoodieArchivedMetaEntry

use of org.apache.hudi.avro.model.HoodieArchivedMetaEntry in project hudi by apache.

the class TestMetadataConversionUtils method testNonEmptyInflightReplace.

@Test
public void testNonEmptyInflightReplace() throws Exception {
    String newCommitTime = HoodieTestTable.makeNewCommitTime();
    createReplace(newCommitTime, WriteOperationType.INSERT_OVERWRITE_TABLE, false);
    HoodieArchivedMetaEntry metaEntry = MetadataConversionUtils.createMetaWrapper(new HoodieInstant(State.INFLIGHT, HoodieTimeline.REPLACE_COMMIT_ACTION, newCommitTime), metaClient);
    assertEquals(metaEntry.getActionState(), State.INFLIGHT.toString());
    assertEquals(metaEntry.getHoodieInflightReplaceMetadata().getOperationType(), WriteOperationType.INSERT_OVERWRITE_TABLE.name());
}

Example 9 with HoodieArchivedMetaEntry

use of org.apache.hudi.avro.model.HoodieArchivedMetaEntry in project hudi by apache.

the class TestMetadataConversionUtils method testCompletedClean.

@Test
public void testCompletedClean() throws Exception {
    String newCommitTime = HoodieTestTable.makeNewCommitTime();
    createCleanMetadata(newCommitTime);
    HoodieArchivedMetaEntry metaEntry = MetadataConversionUtils.createMetaWrapper(new HoodieInstant(State.COMPLETED, HoodieTimeline.CLEAN_ACTION, newCommitTime), metaClient);
    assertEquals(metaEntry.getActionState(), State.COMPLETED.toString());
    assertEquals(metaEntry.getHoodieCleanMetadata().getStartCleanTime(), newCommitTime);
}

Example 10 with HoodieArchivedMetaEntry

use of org.apache.hudi.avro.model.HoodieArchivedMetaEntry in project hudi by apache.

the class MetadataConversionUtils method createMetaWrapper.

public static HoodieArchivedMetaEntry createMetaWrapper(HoodieInstant hoodieInstant, HoodieTableMetaClient metaClient) throws IOException {
    HoodieArchivedMetaEntry archivedMetaWrapper = new HoodieArchivedMetaEntry();
    archivedMetaWrapper.setCommitTime(hoodieInstant.getTimestamp());
    archivedMetaWrapper.setActionState(hoodieInstant.getState().name());
    switch(hoodieInstant.getAction()) {
        case HoodieTimeline.CLEAN_ACTION:
            {
                if (hoodieInstant.isCompleted()) {
                    archivedMetaWrapper.setHoodieCleanMetadata(CleanerUtils.getCleanerMetadata(metaClient, hoodieInstant));
                } else {
                    archivedMetaWrapper.setHoodieCleanerPlan(CleanerUtils.getCleanerPlan(metaClient, hoodieInstant));
                }
                archivedMetaWrapper.setActionType(ActionType.clean.name());
                break;
            }
        case HoodieTimeline.COMMIT_ACTION:
            {
                HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(metaClient.getActiveTimeline().getInstantDetails(hoodieInstant).get(), HoodieCommitMetadata.class);
                archivedMetaWrapper.setHoodieCommitMetadata(convertCommitMetadata(commitMetadata));
                archivedMetaWrapper.setActionType(ActionType.commit.name());
                break;
            }
        case HoodieTimeline.DELTA_COMMIT_ACTION:
            {
                HoodieCommitMetadata deltaCommitMetadata = HoodieCommitMetadata.fromBytes(metaClient.getActiveTimeline().getInstantDetails(hoodieInstant).get(), HoodieCommitMetadata.class);
                archivedMetaWrapper.setHoodieCommitMetadata(convertCommitMetadata(deltaCommitMetadata));
                archivedMetaWrapper.setActionType(ActionType.deltacommit.name());
                break;
            }
        case HoodieTimeline.REPLACE_COMMIT_ACTION:
            {
                if (hoodieInstant.isCompleted()) {
                    HoodieReplaceCommitMetadata replaceCommitMetadata = HoodieReplaceCommitMetadata.fromBytes(metaClient.getActiveTimeline().getInstantDetails(hoodieInstant).get(), HoodieReplaceCommitMetadata.class);
                    archivedMetaWrapper.setHoodieReplaceCommitMetadata(ReplaceArchivalHelper.convertReplaceCommitMetadata(replaceCommitMetadata));
                } else if (hoodieInstant.isInflight()) {
                    // inflight replacecommit files have the same meta data body as HoodieCommitMetadata
                    // so we could re-use it without further creating an inflight extension.
                    // Or inflight replacecommit files are empty under clustering circumstance
                    Option<HoodieCommitMetadata> inflightCommitMetadata = getInflightReplaceMetadata(metaClient, hoodieInstant);
                    if (inflightCommitMetadata.isPresent()) {
                        archivedMetaWrapper.setHoodieInflightReplaceMetadata(convertCommitMetadata(inflightCommitMetadata.get()));
                    }
                } else {
                    // we may have cases with empty HoodieRequestedReplaceMetadata e.g. insert_overwrite_table or insert_overwrite
                    // without clustering. However, we should revisit the requested commit file standardization
                    Option<HoodieRequestedReplaceMetadata> requestedReplaceMetadata = getRequestedReplaceMetadata(metaClient, hoodieInstant);
                    if (requestedReplaceMetadata.isPresent()) {
                        archivedMetaWrapper.setHoodieRequestedReplaceMetadata(requestedReplaceMetadata.get());
                    }
                }
                archivedMetaWrapper.setActionType(ActionType.replacecommit.name());
                break;
            }
        case HoodieTimeline.ROLLBACK_ACTION:
            {
                if (hoodieInstant.isCompleted()) {
                    archivedMetaWrapper.setHoodieRollbackMetadata(TimelineMetadataUtils.deserializeAvroMetadata(metaClient.getActiveTimeline().getInstantDetails(hoodieInstant).get(), HoodieRollbackMetadata.class));
                }
                archivedMetaWrapper.setActionType(ActionType.rollback.name());
                break;
            }
        case HoodieTimeline.SAVEPOINT_ACTION:
            {
                archivedMetaWrapper.setHoodieSavePointMetadata(TimelineMetadataUtils.deserializeAvroMetadata(metaClient.getActiveTimeline().getInstantDetails(hoodieInstant).get(), HoodieSavepointMetadata.class));
                archivedMetaWrapper.setActionType(ActionType.savepoint.name());
                break;
            }
        case HoodieTimeline.COMPACTION_ACTION:
            {
                HoodieCompactionPlan plan = CompactionUtils.getCompactionPlan(metaClient, hoodieInstant.getTimestamp());
                archivedMetaWrapper.setHoodieCompactionPlan(plan);
                archivedMetaWrapper.setActionType(ActionType.compaction.name());
                break;
            }
        default:
            {
                throw new UnsupportedOperationException("Action not fully supported yet");
            }
    }
    return archivedMetaWrapper;
}