Examples with HoodieLogFile org.apache.hudi.common.model.HoodieLogFile used on opensource projects

Example 66 with HoodieLogFile

use of org.apache.hudi.common.model.HoodieLogFile in project hudi by apache.

the class HoodieBackedTableMetadata method openReadersIfNeeded.

/**
 * Create a file reader and the record scanner for a given partition and file slice
 * if readers are not already available.
 *
 * @param partitionName - Partition name
 * @param slice         - The file slice to open readers for
 * @return File reader and the record scanner pair for the requested file slice
 */
private Pair<HoodieFileReader, HoodieMetadataMergedLogRecordReader> openReadersIfNeeded(String partitionName, FileSlice slice) {
    return partitionReaders.computeIfAbsent(Pair.of(partitionName, slice.getFileId()), k -> {
        try {
            HoodieTimer timer = new HoodieTimer().startTimer();
            // Open base file reader
            Pair<HoodieFileReader, Long> baseFileReaderOpenTimePair = getBaseFileReader(slice, timer);
            HoodieFileReader baseFileReader = baseFileReaderOpenTimePair.getKey();
            final long baseFileOpenMs = baseFileReaderOpenTimePair.getValue();
            // Open the log record scanner using the log files from the latest file slice
            List<HoodieLogFile> logFiles = slice.getLogFiles().collect(Collectors.toList());
            Pair<HoodieMetadataMergedLogRecordReader, Long> logRecordScannerOpenTimePair = getLogRecordScanner(logFiles, partitionName);
            HoodieMetadataMergedLogRecordReader logRecordScanner = logRecordScannerOpenTimePair.getKey();
            final long logScannerOpenMs = logRecordScannerOpenTimePair.getValue();
            metrics.ifPresent(metrics -> metrics.updateMetrics(HoodieMetadataMetrics.SCAN_STR, +baseFileOpenMs + logScannerOpenMs));
            return Pair.of(baseFileReader, logRecordScanner);
        } catch (IOException e) {
            throw new HoodieIOException("Error opening readers for metadata table partition " + partitionName, e);
        }
    });
}

Example 67 with HoodieLogFile

use of org.apache.hudi.common.model.HoodieLogFile in project hudi by apache.

the class HoodieMetadataMetrics method getStats.

private Map<String, String> getStats(HoodieTableFileSystemView fsView, boolean detailed, HoodieTableMetadata tableMetadata) throws IOException {
    Map<String, String> stats = new HashMap<>();
    // Total size of the metadata and count of base/log files
    for (String metadataPartition : MetadataPartitionType.allPaths()) {
        List<FileSlice> latestSlices = fsView.getLatestFileSlices(metadataPartition).collect(Collectors.toList());
        // Total size of the metadata and count of base/log files
        long totalBaseFileSizeInBytes = 0;
        long totalLogFileSizeInBytes = 0;
        int baseFileCount = 0;
        int logFileCount = 0;
        for (FileSlice slice : latestSlices) {
            if (slice.getBaseFile().isPresent()) {
                totalBaseFileSizeInBytes += slice.getBaseFile().get().getFileStatus().getLen();
                ++baseFileCount;
            }
            Iterator<HoodieLogFile> it = slice.getLogFiles().iterator();
            while (it.hasNext()) {
                totalLogFileSizeInBytes += it.next().getFileSize();
                ++logFileCount;
            }
        }
        stats.put(metadataPartition + "." + STAT_TOTAL_BASE_FILE_SIZE, String.valueOf(totalBaseFileSizeInBytes));
        stats.put(metadataPartition + "." + STAT_TOTAL_LOG_FILE_SIZE, String.valueOf(totalLogFileSizeInBytes));
        stats.put(metadataPartition + "." + STAT_COUNT_BASE_FILES, String.valueOf(baseFileCount));
        stats.put(metadataPartition + "." + STAT_COUNT_LOG_FILES, String.valueOf(logFileCount));
    }
    if (detailed) {
        stats.put(HoodieMetadataMetrics.STAT_COUNT_PARTITION, String.valueOf(tableMetadata.getAllPartitionPaths().size()));
    }
    return stats;
}

Example 68 with HoodieLogFile

use of org.apache.hudi.common.model.HoodieLogFile in project hudi by apache.

the class ExportCommand method copyArchivedInstants.

private int copyArchivedInstants(List<FileStatus> statuses, Set<String> actionSet, int limit, String localFolder) throws Exception {
    int copyCount = 0;
    for (FileStatus fs : statuses) {
        // read the archived file
        Reader reader = HoodieLogFormat.newReader(FSUtils.getFs(HoodieCLI.getTableMetaClient().getBasePath(), HoodieCLI.conf), new HoodieLogFile(fs.getPath()), HoodieArchivedMetaEntry.getClassSchema());
        // read the avro blocks
        while (reader.hasNext() && copyCount < limit) {
            HoodieAvroDataBlock blk = (HoodieAvroDataBlock) reader.next();
            try (ClosableIterator<IndexedRecord> recordItr = blk.getRecordItr()) {
                while (recordItr.hasNext()) {
                    IndexedRecord ir = recordItr.next();
                    // Archived instants are saved as arvo encoded HoodieArchivedMetaEntry records. We need to get the
                    // metadata record from the entry and convert it to json.
                    HoodieArchivedMetaEntry archiveEntryRecord = (HoodieArchivedMetaEntry) SpecificData.get().deepCopy(HoodieArchivedMetaEntry.SCHEMA$, ir);
                    final String action = archiveEntryRecord.get("actionType").toString();
                    if (!actionSet.contains(action)) {
                        continue;
                    }
                    GenericRecord metadata = null;
                    switch(action) {
                        case HoodieTimeline.CLEAN_ACTION:
                            metadata = archiveEntryRecord.getHoodieCleanMetadata();
                            break;
                        case HoodieTimeline.COMMIT_ACTION:
                        case HoodieTimeline.DELTA_COMMIT_ACTION:
                            metadata = archiveEntryRecord.getHoodieCommitMetadata();
                            break;
                        case HoodieTimeline.ROLLBACK_ACTION:
                            metadata = archiveEntryRecord.getHoodieRollbackMetadata();
                            break;
                        case HoodieTimeline.SAVEPOINT_ACTION:
                            metadata = archiveEntryRecord.getHoodieSavePointMetadata();
                            break;
                        case HoodieTimeline.COMPACTION_ACTION:
                            metadata = archiveEntryRecord.getHoodieCompactionMetadata();
                            break;
                        default:
                            throw new HoodieException("Unknown type of action " + action);
                    }
                    final String instantTime = archiveEntryRecord.get("commitTime").toString();
                    final String outPath = localFolder + Path.SEPARATOR + instantTime + "." + action;
                    writeToFile(outPath, HoodieAvroUtils.avroToJson(metadata, true));
                    if (++copyCount == limit) {
                        break;
                    }
                }
            }
        }
        reader.close();
    }
    return copyCount;
}

Example 69 with HoodieLogFile

use of org.apache.hudi.common.model.HoodieLogFile in project hudi by apache.

the class FileSystemViewCommand method showAllFileSlices.

@CliCommand(value = "show fsview all", help = "Show entire file-system view")
public String showAllFileSlices(@CliOption(key = { "pathRegex" }, help = "regex to select files, eg: 2016/08/02", unspecifiedDefaultValue = "*/*/*") String globRegex, @CliOption(key = { "baseFileOnly" }, help = "Only display base files view", unspecifiedDefaultValue = "false") boolean baseFileOnly, @CliOption(key = { "maxInstant" }, help = "File-Slices upto this instant are displayed", unspecifiedDefaultValue = "") String maxInstant, @CliOption(key = { "includeMax" }, help = "Include Max Instant", unspecifiedDefaultValue = "false") boolean includeMaxInstant, @CliOption(key = { "includeInflight" }, help = "Include Inflight Instants", unspecifiedDefaultValue = "false") boolean includeInflight, @CliOption(key = { "excludeCompaction" }, help = "Exclude compaction Instants", unspecifiedDefaultValue = "false") boolean excludeCompaction, @CliOption(key = { "limit" }, help = "Limit rows to be displayed", unspecifiedDefaultValue = "-1") Integer limit, @CliOption(key = { "sortBy" }, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = { "desc" }, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = { "headeronly" }, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws IOException {
    HoodieTableFileSystemView fsView = buildFileSystemView(globRegex, maxInstant, baseFileOnly, includeMaxInstant, includeInflight, excludeCompaction);
    List<Comparable[]> rows = new ArrayList<>();
    fsView.getAllFileGroups().forEach(fg -> fg.getAllFileSlices().forEach(fs -> {
        int idx = 0;
        // For base file only Views, do not display any delta-file related columns
        Comparable[] row = new Comparable[baseFileOnly ? 5 : 8];
        row[idx++] = fg.getPartitionPath();
        row[idx++] = fg.getFileGroupId().getFileId();
        row[idx++] = fs.getBaseInstantTime();
        row[idx++] = fs.getBaseFile().isPresent() ? fs.getBaseFile().get().getPath() : "";
        row[idx++] = fs.getBaseFile().isPresent() ? fs.getBaseFile().get().getFileSize() : -1;
        if (!baseFileOnly) {
            row[idx++] = fs.getLogFiles().count();
            row[idx++] = fs.getLogFiles().mapToLong(HoodieLogFile::getFileSize).sum();
            row[idx++] = fs.getLogFiles().collect(Collectors.toList()).toString();
        }
        rows.add(row);
    }));
    Function<Object, String> converterFunction = entry -> NumericUtils.humanReadableByteCount((Double.parseDouble(entry.toString())));
    Map<String, Function<Object, String>> fieldNameToConverterMap = new HashMap<>();
    fieldNameToConverterMap.put(HoodieTableHeaderFields.HEADER_TOTAL_DELTA_FILE_SIZE, converterFunction);
    fieldNameToConverterMap.put(HoodieTableHeaderFields.HEADER_DATA_FILE_SIZE, converterFunction);
    TableHeader header = new TableHeader().addTableHeaderField(HoodieTableHeaderFields.HEADER_PARTITION).addTableHeaderField(HoodieTableHeaderFields.HEADER_FILE_ID).addTableHeaderField(HoodieTableHeaderFields.HEADER_BASE_INSTANT).addTableHeaderField(HoodieTableHeaderFields.HEADER_DATA_FILE).addTableHeaderField(HoodieTableHeaderFields.HEADER_DATA_FILE_SIZE);
    if (!baseFileOnly) {
        header = header.addTableHeaderField(HoodieTableHeaderFields.HEADER_NUM_DELTA_FILES).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_DELTA_FILE_SIZE).addTableHeaderField(HoodieTableHeaderFields.HEADER_DELTA_FILES);
    }
    return HoodiePrintHelper.print(header, fieldNameToConverterMap, sortByField, descending, limit, headerOnly, rows);
}

Example 70 with HoodieLogFile

use of org.apache.hudi.common.model.HoodieLogFile in project hudi by apache.

the class HoodieLogFileCommand method showLogFileRecords.

@CliCommand(value = "show logfile records", help = "Read records from log files")
public String showLogFileRecords(@CliOption(key = { "limit" }, help = "Limit commits", unspecifiedDefaultValue = "10") final Integer limit, @CliOption(key = "logFilePathPattern", mandatory = true, help = "Fully qualified paths for the log files") final String logFilePathPattern, @CliOption(key = "mergeRecords", help = "If the records in the log files should be merged", unspecifiedDefaultValue = "false") final Boolean shouldMerge) throws IOException {
    System.out.println("===============> Showing only " + limit + " records <===============");
    HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
    FileSystem fs = client.getFs();
    List<String> logFilePaths = FSUtils.getGlobStatusExcludingMetaFolder(fs, new Path(logFilePathPattern)).stream().map(status -> status.getPath().toString()).sorted(Comparator.reverseOrder()).collect(Collectors.toList());
    // logFilePaths size must > 1
    assert logFilePaths.size() > 0 : "There is no log file";
    // TODO : readerSchema can change across blocks/log files, fix this inside Scanner
    AvroSchemaConverter converter = new AvroSchemaConverter();
    // get schema from last log file
    Schema readerSchema = converter.convert(Objects.requireNonNull(TableSchemaResolver.readSchemaFromLogFile(fs, new Path(logFilePaths.get(logFilePaths.size() - 1)))));
    List<IndexedRecord> allRecords = new ArrayList<>();
    if (shouldMerge) {
        System.out.println("===========================> MERGING RECORDS <===================");
        HoodieMergedLogRecordScanner scanner = HoodieMergedLogRecordScanner.newBuilder().withFileSystem(fs).withBasePath(client.getBasePath()).withLogFilePaths(logFilePaths).withReaderSchema(readerSchema).withLatestInstantTime(client.getActiveTimeline().getCommitTimeline().lastInstant().get().getTimestamp()).withReadBlocksLazily(Boolean.parseBoolean(HoodieCompactionConfig.COMPACTION_LAZY_BLOCK_READ_ENABLE.defaultValue())).withReverseReader(Boolean.parseBoolean(HoodieCompactionConfig.COMPACTION_REVERSE_LOG_READ_ENABLE.defaultValue())).withBufferSize(HoodieMemoryConfig.MAX_DFS_STREAM_BUFFER_SIZE.defaultValue()).withMaxMemorySizeInBytes(HoodieMemoryConfig.DEFAULT_MAX_MEMORY_FOR_SPILLABLE_MAP_IN_BYTES).withSpillableMapBasePath(HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH.defaultValue()).withDiskMapType(HoodieCommonConfig.SPILLABLE_DISK_MAP_TYPE.defaultValue()).withBitCaskDiskMapCompressionEnabled(HoodieCommonConfig.DISK_MAP_BITCASK_COMPRESSION_ENABLED.defaultValue()).build();
        for (HoodieRecord<? extends HoodieRecordPayload> hoodieRecord : scanner) {
            Option<IndexedRecord> record = hoodieRecord.getData().getInsertValue(readerSchema);
            if (allRecords.size() < limit) {
                allRecords.add(record.get());
            }
        }
    } else {
        for (String logFile : logFilePaths) {
            Schema writerSchema = new AvroSchemaConverter().convert(Objects.requireNonNull(TableSchemaResolver.readSchemaFromLogFile(client.getFs(), new Path(logFile))));
            HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(fs, new HoodieLogFile(new Path(logFile)), writerSchema);
            // read the avro blocks
            while (reader.hasNext()) {
                HoodieLogBlock n = reader.next();
                if (n instanceof HoodieDataBlock) {
                    HoodieDataBlock blk = (HoodieDataBlock) n;
                    try (ClosableIterator<IndexedRecord> recordItr = blk.getRecordItr()) {
                        recordItr.forEachRemaining(record -> {
                            if (allRecords.size() < limit) {
                                allRecords.add(record);
                            }
                        });
                    }
                }
            }
            reader.close();
            if (allRecords.size() >= limit) {
                break;
            }
        }
    }
    String[][] rows = new String[allRecords.size()][];
    int i = 0;
    for (IndexedRecord record : allRecords) {
        String[] data = new String[1];
        data[0] = record.toString();
        rows[i] = data;
        i++;
    }
    return HoodiePrintHelper.print(new String[] { HoodieTableHeaderFields.HEADER_RECORDS }, rows);
}