Examples with TableHeader org.apache.hudi.cli.TableHeader used on opensource projects

Example 21 with TableHeader

use of org.apache.hudi.cli.TableHeader in project hudi by apache.

the class CleansCommand method showCleanPartitions.

@CliCommand(value = "clean showpartitions", help = "Show partition level details of a clean")
public String showCleanPartitions(@CliOption(key = { "clean" }, help = "clean to show") final String instantTime, @CliOption(key = { "limit" }, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit, @CliOption(key = { "sortBy" }, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = { "desc" }, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = { "headeronly" }, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws Exception {
    HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
    HoodieTimeline timeline = activeTimeline.getCleanerTimeline().filterCompletedInstants();
    HoodieInstant cleanInstant = new HoodieInstant(false, HoodieTimeline.CLEAN_ACTION, instantTime);
    if (!timeline.containsInstant(cleanInstant)) {
        return "Clean " + instantTime + " not found in metadata " + timeline;
    }
    HoodieCleanMetadata cleanMetadata = TimelineMetadataUtils.deserializeHoodieCleanMetadata(timeline.getInstantDetails(cleanInstant).get());
    List<Comparable[]> rows = new ArrayList<>();
    for (Map.Entry<String, HoodieCleanPartitionMetadata> entry : cleanMetadata.getPartitionMetadata().entrySet()) {
        String path = entry.getKey();
        HoodieCleanPartitionMetadata stats = entry.getValue();
        String policy = stats.getPolicy();
        int totalSuccessDeletedFiles = stats.getSuccessDeleteFiles().size();
        int totalFailedDeletedFiles = stats.getFailedDeleteFiles().size();
        rows.add(new Comparable[] { path, policy, totalSuccessDeletedFiles, totalFailedDeletedFiles });
    }
    TableHeader header = new TableHeader().addTableHeaderField(HoodieTableHeaderFields.HEADER_PARTITION_PATH).addTableHeaderField(HoodieTableHeaderFields.HEADER_CLEANING_POLICY).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_FILES_SUCCESSFULLY_DELETED).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_FAILED_DELETIONS);
    return HoodiePrintHelper.print(header, new HashMap<>(), sortByField, descending, limit, headerOnly, rows);
}

Example 22 with TableHeader

use of org.apache.hudi.cli.TableHeader in project hudi by apache.

the class TestArchivedCommitsCommand method testShowArchivedCommits.

/**
 * Test for command: show archived commit stats.
 */
@Test
public void testShowArchivedCommits() {
    CommandResult cr = shell().executeCommand("show archived commit stats");
    assertTrue(cr.isSuccess());
    TableHeader header = new TableHeader().addTableHeaderField("action").addTableHeaderField("instant").addTableHeaderField("partition").addTableHeaderField("file_id").addTableHeaderField("prev_instant").addTableHeaderField("num_writes").addTableHeaderField("num_inserts").addTableHeaderField("num_deletes").addTableHeaderField("num_update_writes").addTableHeaderField("total_log_files").addTableHeaderField("total_log_blocks").addTableHeaderField("total_corrupt_log_blocks").addTableHeaderField("total_rollback_blocks").addTableHeaderField("total_log_records").addTableHeaderField("total_updated_records_compacted").addTableHeaderField("total_write_bytes").addTableHeaderField("total_write_errors");
    // Generate expected data
    final List<Comparable[]> rows = new ArrayList<>();
    for (int i = 100; i < 104; i++) {
        String instant = String.valueOf(i);
        for (int j = 0; j < 3; j++) {
            Comparable[] defaultComp = new Comparable[] { "commit", instant, HoodieTestCommitMetadataGenerator.DEFAULT_SECOND_PARTITION_PATH, HoodieTestCommitMetadataGenerator.DEFAULT_FILEID, HoodieTestCommitMetadataGenerator.DEFAULT_PRE_COMMIT, HoodieTestCommitMetadataGenerator.DEFAULT_NUM_WRITES, HoodieTestCommitMetadataGenerator.DEFAULT_OTHER_VALUE, HoodieTestCommitMetadataGenerator.DEFAULT_OTHER_VALUE, HoodieTestCommitMetadataGenerator.DEFAULT_NUM_UPDATE_WRITES, HoodieTestCommitMetadataGenerator.DEFAULT_NULL_VALUE, HoodieTestCommitMetadataGenerator.DEFAULT_TOTAL_LOG_BLOCKS, HoodieTestCommitMetadataGenerator.DEFAULT_OTHER_VALUE, HoodieTestCommitMetadataGenerator.DEFAULT_OTHER_VALUE, HoodieTestCommitMetadataGenerator.DEFAULT_TOTAL_LOG_RECORDS, HoodieTestCommitMetadataGenerator.DEFAULT_OTHER_VALUE, HoodieTestCommitMetadataGenerator.DEFAULT_TOTAL_WRITE_BYTES, HoodieTestCommitMetadataGenerator.DEFAULT_OTHER_VALUE };
            rows.add(defaultComp.clone());
            defaultComp[2] = HoodieTestCommitMetadataGenerator.DEFAULT_FIRST_PARTITION_PATH;
            rows.add(defaultComp);
        }
    }
    String expectedResult = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows);
    expectedResult = removeNonWordAndStripSpace(expectedResult);
    String got = removeNonWordAndStripSpace(cr.getResult().toString());
    assertEquals(expectedResult, got);
}

Example 23 with TableHeader

use of org.apache.hudi.cli.TableHeader in project hudi by apache.

the class CommitsCommand method showCommitFiles.

@CliCommand(value = "commit showfiles", help = "Show file level details of a commit")
public String showCommitFiles(@CliOption(key = { "createView" }, mandatory = false, help = "view name to store output table", unspecifiedDefaultValue = "") final String exportTableName, @CliOption(key = { "commit" }, help = "Commit to show") final String instantTime, @CliOption(key = { "limit" }, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit, @CliOption(key = { "sortBy" }, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = { "desc" }, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = { "headeronly" }, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws Exception {
    HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
    HoodieTimeline timeline = activeTimeline.getCommitsTimeline().filterCompletedInstants();
    Option<HoodieInstant> hoodieInstantOption = getCommitForInstant(timeline, instantTime);
    Option<HoodieCommitMetadata> commitMetadataOptional = getHoodieCommitMetadata(timeline, hoodieInstantOption);
    if (!commitMetadataOptional.isPresent()) {
        return "Commit " + instantTime + " not found in Commits " + timeline;
    }
    HoodieCommitMetadata meta = commitMetadataOptional.get();
    List<Comparable[]> rows = new ArrayList<>();
    for (Map.Entry<String, List<HoodieWriteStat>> entry : meta.getPartitionToWriteStats().entrySet()) {
        String action = hoodieInstantOption.get().getAction();
        String path = entry.getKey();
        List<HoodieWriteStat> stats = entry.getValue();
        for (HoodieWriteStat stat : stats) {
            rows.add(new Comparable[] { action, path, stat.getFileId(), stat.getPrevCommit(), stat.getNumUpdateWrites(), stat.getNumWrites(), stat.getTotalWriteBytes(), stat.getTotalWriteErrors(), stat.getFileSizeInBytes() });
        }
    }
    TableHeader header = new TableHeader().addTableHeaderField(HoodieTableHeaderFields.HEADER_ACTION).addTableHeaderField(HoodieTableHeaderFields.HEADER_PARTITION_PATH).addTableHeaderField(HoodieTableHeaderFields.HEADER_FILE_ID).addTableHeaderField(HoodieTableHeaderFields.HEADER_PREVIOUS_COMMIT).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_RECORDS_UPDATED).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_RECORDS_WRITTEN).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_BYTES_WRITTEN).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_ERRORS).addTableHeaderField(HoodieTableHeaderFields.HEADER_FILE_SIZE);
    return HoodiePrintHelper.print(header, new HashMap<>(), sortByField, descending, limit, headerOnly, rows, exportTableName);
}

Example 24 with TableHeader

use of org.apache.hudi.cli.TableHeader in project hudi by apache.

the class CommitsCommand method printCommitsWithMetadata.

private String printCommitsWithMetadata(HoodieDefaultTimeline timeline, final Integer limit, final String sortByField, final boolean descending, final boolean headerOnly, final String tempTableName) throws IOException {
    final List<Comparable[]> rows = new ArrayList<>();
    final List<HoodieInstant> commits = timeline.getCommitsTimeline().filterCompletedInstants().getInstants().collect(Collectors.toList());
    // timeline can be read from multiple files. So sort is needed instead of reversing the collection
    Collections.sort(commits, HoodieInstant.COMPARATOR.reversed());
    for (int i = 0; i < commits.size(); i++) {
        final HoodieInstant commit = commits.get(i);
        final HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(timeline.getInstantDetails(commit).get(), HoodieCommitMetadata.class);
        for (Map.Entry<String, List<HoodieWriteStat>> partitionWriteStat : commitMetadata.getPartitionToWriteStats().entrySet()) {
            for (HoodieWriteStat hoodieWriteStat : partitionWriteStat.getValue()) {
                rows.add(new Comparable[] { commit.getAction(), commit.getTimestamp(), hoodieWriteStat.getPartitionPath(), hoodieWriteStat.getFileId(), hoodieWriteStat.getPrevCommit(), hoodieWriteStat.getNumWrites(), hoodieWriteStat.getNumInserts(), hoodieWriteStat.getNumDeletes(), hoodieWriteStat.getNumUpdateWrites(), hoodieWriteStat.getTotalWriteErrors(), hoodieWriteStat.getTotalLogBlocks(), hoodieWriteStat.getTotalCorruptLogBlock(), hoodieWriteStat.getTotalRollbackBlocks(), hoodieWriteStat.getTotalLogRecords(), hoodieWriteStat.getTotalUpdatedRecordsCompacted(), hoodieWriteStat.getTotalWriteBytes() });
            }
        }
    }
    final Map<String, Function<Object, String>> fieldNameToConverterMap = new HashMap<>();
    fieldNameToConverterMap.put(HoodieTableHeaderFields.HEADER_TOTAL_BYTES_WRITTEN, entry -> {
        return NumericUtils.humanReadableByteCount((Double.valueOf(entry.toString())));
    });
    TableHeader header = new TableHeader().addTableHeaderField(HoodieTableHeaderFields.HEADER_ACTION).addTableHeaderField(HoodieTableHeaderFields.HEADER_INSTANT).addTableHeaderField(HoodieTableHeaderFields.HEADER_PARTITION).addTableHeaderField(HoodieTableHeaderFields.HEADER_FILE_ID).addTableHeaderField(HoodieTableHeaderFields.HEADER_PREVIOUS_COMMIT).addTableHeaderField(HoodieTableHeaderFields.HEADER_NUM_WRITES).addTableHeaderField(HoodieTableHeaderFields.HEADER_NUM_INSERTS).addTableHeaderField(HoodieTableHeaderFields.HEADER_NUM_DELETES).addTableHeaderField(HoodieTableHeaderFields.HEADER_NUM_UPDATE_WRITES).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_ERRORS).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_LOG_BLOCKS).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_CORRUPT_LOG_BLOCKS).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_ROLLBACK_BLOCKS).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_LOG_RECORDS).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_UPDATED_RECORDS_COMPACTED).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_BYTES_WRITTEN);
    return HoodiePrintHelper.print(header, new HashMap<>(), sortByField, descending, limit, headerOnly, rows, tempTableName);
}

Example 25 with TableHeader

use of org.apache.hudi.cli.TableHeader in project hudi by apache.

the class CommitsCommand method printCommits.

private String printCommits(HoodieDefaultTimeline timeline, final Integer limit, final String sortByField, final boolean descending, final boolean headerOnly, final String tempTableName) throws IOException {
    final List<Comparable[]> rows = new ArrayList<>();
    final List<HoodieInstant> commits = timeline.getCommitsTimeline().filterCompletedInstants().getInstants().collect(Collectors.toList());
    // timeline can be read from multiple files. So sort is needed instead of reversing the collection
    Collections.sort(commits, HoodieInstant.COMPARATOR.reversed());
    for (int i = 0; i < commits.size(); i++) {
        final HoodieInstant commit = commits.get(i);
        final HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(timeline.getInstantDetails(commit).get(), HoodieCommitMetadata.class);
        rows.add(new Comparable[] { commit.getTimestamp(), commitMetadata.fetchTotalBytesWritten(), commitMetadata.fetchTotalFilesInsert(), commitMetadata.fetchTotalFilesUpdated(), commitMetadata.fetchTotalPartitionsWritten(), commitMetadata.fetchTotalRecordsWritten(), commitMetadata.fetchTotalUpdateRecordsWritten(), commitMetadata.fetchTotalWriteErrors() });
    }
    final Map<String, Function<Object, String>> fieldNameToConverterMap = new HashMap<>();
    fieldNameToConverterMap.put(HoodieTableHeaderFields.HEADER_TOTAL_BYTES_WRITTEN, entry -> {
        return NumericUtils.humanReadableByteCount((Double.valueOf(entry.toString())));
    });
    final TableHeader header = new TableHeader().addTableHeaderField(HoodieTableHeaderFields.HEADER_COMMIT_TIME).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_BYTES_WRITTEN).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_FILES_ADDED).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_FILES_UPDATED).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_PARTITIONS_WRITTEN).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_RECORDS_WRITTEN).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_UPDATE_RECORDS_WRITTEN).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_ERRORS);
    return HoodiePrintHelper.print(header, fieldNameToConverterMap, sortByField, descending, limit, headerOnly, rows, tempTableName);
}