Examples with HoodieArchivedMetaEntry org.apache.hudi.avro.model.HoodieArchivedMetaEntry used on opensource projects

Example 11 with HoodieArchivedMetaEntry

use of org.apache.hudi.avro.model.HoodieArchivedMetaEntry in project hudi by apache.

the class ExportCommand method copyArchivedInstants.

private int copyArchivedInstants(List<FileStatus> statuses, Set<String> actionSet, int limit, String localFolder) throws Exception {
    int copyCount = 0;
    for (FileStatus fs : statuses) {
        // read the archived file
        Reader reader = HoodieLogFormat.newReader(FSUtils.getFs(HoodieCLI.getTableMetaClient().getBasePath(), HoodieCLI.conf), new HoodieLogFile(fs.getPath()), HoodieArchivedMetaEntry.getClassSchema());
        // read the avro blocks
        while (reader.hasNext() && copyCount < limit) {
            HoodieAvroDataBlock blk = (HoodieAvroDataBlock) reader.next();
            try (ClosableIterator<IndexedRecord> recordItr = blk.getRecordItr()) {
                while (recordItr.hasNext()) {
                    IndexedRecord ir = recordItr.next();
                    // Archived instants are saved as arvo encoded HoodieArchivedMetaEntry records. We need to get the
                    // metadata record from the entry and convert it to json.
                    HoodieArchivedMetaEntry archiveEntryRecord = (HoodieArchivedMetaEntry) SpecificData.get().deepCopy(HoodieArchivedMetaEntry.SCHEMA$, ir);
                    final String action = archiveEntryRecord.get("actionType").toString();
                    if (!actionSet.contains(action)) {
                        continue;
                    }
                    GenericRecord metadata = null;
                    switch(action) {
                        case HoodieTimeline.CLEAN_ACTION:
                            metadata = archiveEntryRecord.getHoodieCleanMetadata();
                            break;
                        case HoodieTimeline.COMMIT_ACTION:
                        case HoodieTimeline.DELTA_COMMIT_ACTION:
                            metadata = archiveEntryRecord.getHoodieCommitMetadata();
                            break;
                        case HoodieTimeline.ROLLBACK_ACTION:
                            metadata = archiveEntryRecord.getHoodieRollbackMetadata();
                            break;
                        case HoodieTimeline.SAVEPOINT_ACTION:
                            metadata = archiveEntryRecord.getHoodieSavePointMetadata();
                            break;
                        case HoodieTimeline.COMPACTION_ACTION:
                            metadata = archiveEntryRecord.getHoodieCompactionMetadata();
                            break;
                        default:
                            throw new HoodieException("Unknown type of action " + action);
                    }
                    final String instantTime = archiveEntryRecord.get("commitTime").toString();
                    final String outPath = localFolder + Path.SEPARATOR + instantTime + "." + action;
                    writeToFile(outPath, HoodieAvroUtils.avroToJson(metadata, true));
                    if (++copyCount == limit) {
                        break;
                    }
                }
            }
        }
        reader.close();
    }
    return copyCount;
}