Examples with TableHeader org.apache.hudi.cli.TableHeader used on opensource projects

Example 41 with TableHeader

use of org.apache.hudi.cli.TableHeader in project hudi by apache.

the class TestArchivedCommitsCommand method testShowCommits.

/**
 * Test for command: show archived commits.
 */
@Test
public void testShowCommits() throws Exception {
    CommandResult cr = shell().executeCommand("show archived commits");
    assertTrue(cr.isSuccess());
    final List<Comparable[]> rows = new ArrayList<>();
    // Test default skipMetadata and limit 10
    TableHeader header = new TableHeader().addTableHeaderField("CommitTime").addTableHeaderField("CommitType");
    for (int i = 100; i < 103; i++) {
        String instant = String.valueOf(i);
        Comparable[] result = new Comparable[] { instant, "commit" };
        rows.add(result);
        rows.add(result);
        rows.add(result);
    }
    rows.add(new Comparable[] { "103", "commit" });
    String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, 10, false, rows);
    expected = removeNonWordAndStripSpace(expected);
    String got = removeNonWordAndStripSpace(cr.getResult().toString());
    assertEquals(expected, got);
    // Test with Metadata and no limit
    cr = shell().executeCommand("show archived commits --skipMetadata false --limit -1");
    assertTrue(cr.isSuccess());
    rows.clear();
    for (int i = 100; i < 104; i++) {
        String instant = String.valueOf(i);
        // Since HoodiePrintHelper order data by default, need to order commitMetadata
        HoodieCommitMetadata metadata = HoodieTestCommitMetadataGenerator.generateCommitMetadata(tablePath, instant);
        Comparable[] result = new Comparable[] { instant, "commit", HoodieTestCommitUtilities.convertAndOrderCommitMetadata(metadata) };
        rows.add(result);
        rows.add(result);
        rows.add(result);
    }
    header = header.addTableHeaderField("CommitDetails");
    expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows);
    expected = removeNonWordAndStripSpace(expected);
    got = removeNonWordAndStripSpace(cr.getResult().toString());
    assertEquals(expected, got);
}

Example 42 with TableHeader

use of org.apache.hudi.cli.TableHeader in project hudi by apache.

the class TestCleansCommand method testShowCleans.

/**
 * Test case for show all cleans.
 */
@Test
public void testShowCleans() throws Exception {
    // Check properties file exists.
    assertNotNull(propsFilePath, "Not found properties file");
    // First, run clean
    SparkMain.clean(jsc(), HoodieCLI.basePath, propsFilePath.getPath(), new ArrayList<>());
    assertEquals(1, metaClient.getActiveTimeline().reload().getCleanerTimeline().getInstants().count(), "Loaded 1 clean and the count should match");
    CommandResult cr = shell().executeCommand("cleans show");
    assertTrue(cr.isSuccess());
    HoodieInstant clean = metaClient.getActiveTimeline().reload().getCleanerTimeline().getInstants().findFirst().orElse(null);
    assertNotNull(clean);
    TableHeader header = new TableHeader().addTableHeaderField(HoodieTableHeaderFields.HEADER_CLEAN_TIME).addTableHeaderField(HoodieTableHeaderFields.HEADER_EARLIEST_COMMAND_RETAINED).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_FILES_DELETED).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_TIME_TAKEN);
    List<Comparable[]> rows = new ArrayList<>();
    // EarliestCommandRetained should be 102, since hoodie.cleaner.commits.retained=2
    // Total Time Taken need read from metadata
    rows.add(new Comparable[] { clean.getTimestamp(), "102", "2", getLatestCleanTimeTakenInMillis().toString() });
    String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows);
    expected = removeNonWordAndStripSpace(expected);
    String got = removeNonWordAndStripSpace(cr.getResult().toString());
    assertEquals(expected, got);
}

Example 43 with TableHeader

use of org.apache.hudi.cli.TableHeader in project hudi by apache.

the class TestCleansCommand method testShowCleanPartitions.

/**
 * Test case for show partitions of a clean instant.
 */
@Test
public void testShowCleanPartitions() {
    // Check properties file exists.
    assertNotNull(propsFilePath, "Not found properties file");
    // First, run clean with two partition
    SparkMain.clean(jsc(), HoodieCLI.basePath, propsFilePath.toString(), new ArrayList<>());
    assertEquals(1, metaClient.getActiveTimeline().reload().getCleanerTimeline().getInstants().count(), "Loaded 1 clean and the count should match");
    HoodieInstant clean = metaClient.getActiveTimeline().reload().getCleanerTimeline().getInstants().findFirst().get();
    CommandResult cr = shell().executeCommand("clean showpartitions --clean " + clean.getTimestamp());
    assertTrue(cr.isSuccess());
    TableHeader header = new TableHeader().addTableHeaderField(HoodieTableHeaderFields.HEADER_PARTITION_PATH).addTableHeaderField(HoodieTableHeaderFields.HEADER_CLEANING_POLICY).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_FILES_SUCCESSFULLY_DELETED).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_FAILED_DELETIONS);
    // There should be two partition path
    List<Comparable[]> rows = new ArrayList<>();
    rows.add(new Comparable[] { HoodieTestCommitMetadataGenerator.DEFAULT_SECOND_PARTITION_PATH, HoodieCleaningPolicy.KEEP_LATEST_COMMITS, "1", "0" });
    rows.add(new Comparable[] { HoodieTestCommitMetadataGenerator.DEFAULT_THIRD_PARTITION_PATH, HoodieCleaningPolicy.KEEP_LATEST_COMMITS, "0", "0" });
    rows.add(new Comparable[] { HoodieTestCommitMetadataGenerator.DEFAULT_FIRST_PARTITION_PATH, HoodieCleaningPolicy.KEEP_LATEST_COMMITS, "1", "0" });
    String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows);
    expected = removeNonWordAndStripSpace(expected);
    String got = removeNonWordAndStripSpace(cr.getResult().toString());
    assertEquals(expected, got);
}

Example 44 with TableHeader

use of org.apache.hudi.cli.TableHeader in project hudi by apache.

the class ArchivedCommitsCommand method showCommits.

@CliCommand(value = "show archived commits", help = "Read commits from archived files and show details")
public String showCommits(@CliOption(key = { "skipMetadata" }, help = "Skip displaying commit metadata", unspecifiedDefaultValue = "true") boolean skipMetadata, @CliOption(key = { "limit" }, help = "Limit commits", unspecifiedDefaultValue = "10") final Integer limit, @CliOption(key = { "sortBy" }, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = { "desc" }, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = { "headeronly" }, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws IOException {
    System.out.println("===============> Showing only " + limit + " archived commits <===============");
    HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
    String basePath = metaClient.getBasePath();
    Path archivePath = new Path(metaClient.getArchivePath() + "/.commits_.archive*");
    FileStatus[] fsStatuses = FSUtils.getFs(basePath, HoodieCLI.conf).globStatus(archivePath);
    List<Comparable[]> allCommits = new ArrayList<>();
    for (FileStatus fs : fsStatuses) {
        // read the archived file
        HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(FSUtils.getFs(basePath, HoodieCLI.conf), new HoodieLogFile(fs.getPath()), HoodieArchivedMetaEntry.getClassSchema());
        List<IndexedRecord> readRecords = new ArrayList<>();
        // read the avro blocks
        while (reader.hasNext()) {
            HoodieAvroDataBlock blk = (HoodieAvroDataBlock) reader.next();
            try (ClosableIterator<IndexedRecord> recordItr = blk.getRecordItr()) {
                recordItr.forEachRemaining(readRecords::add);
            }
        }
        List<Comparable[]> readCommits = readRecords.stream().map(r -> (GenericRecord) r).map(r -> readCommit(r, skipMetadata)).collect(Collectors.toList());
        allCommits.addAll(readCommits);
        reader.close();
    }
    TableHeader header = new TableHeader().addTableHeaderField("CommitTime").addTableHeaderField("CommitType");
    if (!skipMetadata) {
        header = header.addTableHeaderField("CommitDetails");
    }
    return HoodiePrintHelper.print(header, new HashMap<>(), sortByField, descending, limit, headerOnly, allCommits);
}

Example 45 with TableHeader

use of org.apache.hudi.cli.TableHeader in project hudi by apache.

the class CleansCommand method showCleans.

@CliCommand(value = "cleans show", help = "Show the cleans")
public String showCleans(@CliOption(key = { "limit" }, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit, @CliOption(key = { "sortBy" }, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = { "desc" }, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = { "headeronly" }, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws IOException {
    HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
    HoodieTimeline timeline = activeTimeline.getCleanerTimeline().filterCompletedInstants();
    List<HoodieInstant> cleans = timeline.getReverseOrderedInstants().collect(Collectors.toList());
    List<Comparable[]> rows = new ArrayList<>();
    for (HoodieInstant clean : cleans) {
        HoodieCleanMetadata cleanMetadata = TimelineMetadataUtils.deserializeHoodieCleanMetadata(timeline.getInstantDetails(clean).get());
        rows.add(new Comparable[] { clean.getTimestamp(), cleanMetadata.getEarliestCommitToRetain(), cleanMetadata.getTotalFilesDeleted(), cleanMetadata.getTimeTakenInMillis() });
    }
    TableHeader header = new TableHeader().addTableHeaderField(HoodieTableHeaderFields.HEADER_CLEAN_TIME).addTableHeaderField(HoodieTableHeaderFields.HEADER_EARLIEST_COMMAND_RETAINED).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_FILES_DELETED).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_TIME_TAKEN);
    return HoodiePrintHelper.print(header, new HashMap<>(), sortByField, descending, limit, headerOnly, rows);
}