Examples with ClosableIterator org.apache.hudi.common.util.ClosableIterator used on opensource projects

Example 6 with ClosableIterator

use of org.apache.hudi.common.util.ClosableIterator in project hudi by apache.

the class HoodieHFileDataBlock method deserializeRecords.

@Override
protected ClosableIterator<IndexedRecord> deserializeRecords(byte[] content) throws IOException {
    checkState(readerSchema != null, "Reader's schema has to be non-null");
    // Get schema from the header
    Schema writerSchema = new Schema.Parser().parse(super.getLogBlockHeader().get(HeaderMetadataType.SCHEMA));
    // Read the content
    HoodieHFileReader<IndexedRecord> reader = new HoodieHFileReader<>(content);
    // Sets up the writer schema
    reader.withSchema(writerSchema);
    Iterator<IndexedRecord> recordIterator = reader.getRecordIterator(readerSchema);
    return new ClosableIterator<IndexedRecord>() {

        @Override
        public void close() {
            reader.close();
        }

        @Override
        public boolean hasNext() {
            return recordIterator.hasNext();
        }

        @Override
        public IndexedRecord next() {
            return recordIterator.next();
        }
    };
}

Example 7 with ClosableIterator

use of org.apache.hudi.common.util.ClosableIterator in project hudi by apache.

the class HoodieLogFileCommand method showLogFileCommits.

@CliCommand(value = "show logfile metadata", help = "Read commit metadata from log files")
public String showLogFileCommits(@CliOption(key = "logFilePathPattern", mandatory = true, help = "Fully qualified path for the log file") final String logFilePathPattern, @CliOption(key = { "limit" }, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit, @CliOption(key = { "sortBy" }, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = { "desc" }, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = { "headeronly" }, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws IOException {
    FileSystem fs = HoodieCLI.getTableMetaClient().getFs();
    List<String> logFilePaths = FSUtils.getGlobStatusExcludingMetaFolder(fs, new Path(logFilePathPattern)).stream().map(status -> status.getPath().toString()).collect(Collectors.toList());
    Map<String, List<Tuple3<HoodieLogBlockType, Tuple2<Map<HeaderMetadataType, String>, Map<HeaderMetadataType, String>>, Integer>>> commitCountAndMetadata = new HashMap<>();
    int numCorruptBlocks = 0;
    int dummyInstantTimeCount = 0;
    for (String logFilePath : logFilePaths) {
        FileStatus[] fsStatus = fs.listStatus(new Path(logFilePath));
        Schema writerSchema = new AvroSchemaConverter().convert(Objects.requireNonNull(TableSchemaResolver.readSchemaFromLogFile(fs, new Path(logFilePath))));
        Reader reader = HoodieLogFormat.newReader(fs, new HoodieLogFile(fsStatus[0].getPath()), writerSchema);
        // read the avro blocks
        while (reader.hasNext()) {
            HoodieLogBlock n = reader.next();
            String instantTime;
            AtomicInteger recordCount = new AtomicInteger(0);
            if (n instanceof HoodieCorruptBlock) {
                try {
                    instantTime = n.getLogBlockHeader().get(HeaderMetadataType.INSTANT_TIME);
                    if (instantTime == null) {
                        throw new Exception("Invalid instant time " + instantTime);
                    }
                } catch (Exception e) {
                    numCorruptBlocks++;
                    instantTime = "corrupt_block_" + numCorruptBlocks;
                // could not read metadata for corrupt block
                }
            } else {
                instantTime = n.getLogBlockHeader().get(HeaderMetadataType.INSTANT_TIME);
                if (instantTime == null) {
                    // This can happen when reading archived commit files since they were written without any instant time
                    dummyInstantTimeCount++;
                    instantTime = "dummy_instant_time_" + dummyInstantTimeCount;
                }
                if (n instanceof HoodieDataBlock) {
                    try (ClosableIterator<IndexedRecord> recordItr = ((HoodieDataBlock) n).getRecordItr()) {
                        recordItr.forEachRemaining(r -> recordCount.incrementAndGet());
                    }
                }
            }
            if (commitCountAndMetadata.containsKey(instantTime)) {
                commitCountAndMetadata.get(instantTime).add(new Tuple3<>(n.getBlockType(), new Tuple2<>(n.getLogBlockHeader(), n.getLogBlockFooter()), recordCount.get()));
            } else {
                List<Tuple3<HoodieLogBlockType, Tuple2<Map<HeaderMetadataType, String>, Map<HeaderMetadataType, String>>, Integer>> list = new ArrayList<>();
                list.add(new Tuple3<>(n.getBlockType(), new Tuple2<>(n.getLogBlockHeader(), n.getLogBlockFooter()), recordCount.get()));
                commitCountAndMetadata.put(instantTime, list);
            }
        }
        reader.close();
    }
    List<Comparable[]> rows = new ArrayList<>();
    ObjectMapper objectMapper = new ObjectMapper();
    for (Map.Entry<String, List<Tuple3<HoodieLogBlockType, Tuple2<Map<HeaderMetadataType, String>, Map<HeaderMetadataType, String>>, Integer>>> entry : commitCountAndMetadata.entrySet()) {
        String instantTime = entry.getKey();
        for (Tuple3<HoodieLogBlockType, Tuple2<Map<HeaderMetadataType, String>, Map<HeaderMetadataType, String>>, Integer> tuple3 : entry.getValue()) {
            Comparable[] output = new Comparable[5];
            output[0] = instantTime;
            output[1] = tuple3._3();
            output[2] = tuple3._1().toString();
            output[3] = objectMapper.writeValueAsString(tuple3._2()._1());
            output[4] = objectMapper.writeValueAsString(tuple3._2()._2());
            rows.add(output);
        }
    }
    TableHeader header = new TableHeader().addTableHeaderField(HoodieTableHeaderFields.HEADER_INSTANT_TIME).addTableHeaderField(HoodieTableHeaderFields.HEADER_RECORD_COUNT).addTableHeaderField(HoodieTableHeaderFields.HEADER_BLOCK_TYPE).addTableHeaderField(HoodieTableHeaderFields.HEADER_HEADER_METADATA).addTableHeaderField(HoodieTableHeaderFields.HEADER_FOOTER_METADATA);
    return HoodiePrintHelper.print(header, new HashMap<>(), sortByField, descending, limit, headerOnly, rows);
}

Example 8 with ClosableIterator

use of org.apache.hudi.common.util.ClosableIterator in project hudi by apache.

the class ArchivedCommitsCommand method showCommits.

@CliCommand(value = "show archived commits", help = "Read commits from archived files and show details")
public String showCommits(@CliOption(key = { "skipMetadata" }, help = "Skip displaying commit metadata", unspecifiedDefaultValue = "true") boolean skipMetadata, @CliOption(key = { "limit" }, help = "Limit commits", unspecifiedDefaultValue = "10") final Integer limit, @CliOption(key = { "sortBy" }, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = { "desc" }, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = { "headeronly" }, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws IOException {
    System.out.println("===============> Showing only " + limit + " archived commits <===============");
    HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
    String basePath = metaClient.getBasePath();
    Path archivePath = new Path(metaClient.getArchivePath() + "/.commits_.archive*");
    FileStatus[] fsStatuses = FSUtils.getFs(basePath, HoodieCLI.conf).globStatus(archivePath);
    List<Comparable[]> allCommits = new ArrayList<>();
    for (FileStatus fs : fsStatuses) {
        // read the archived file
        HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(FSUtils.getFs(basePath, HoodieCLI.conf), new HoodieLogFile(fs.getPath()), HoodieArchivedMetaEntry.getClassSchema());
        List<IndexedRecord> readRecords = new ArrayList<>();
        // read the avro blocks
        while (reader.hasNext()) {
            HoodieAvroDataBlock blk = (HoodieAvroDataBlock) reader.next();
            try (ClosableIterator<IndexedRecord> recordItr = blk.getRecordItr()) {
                recordItr.forEachRemaining(readRecords::add);
            }
        }
        List<Comparable[]> readCommits = readRecords.stream().map(r -> (GenericRecord) r).map(r -> readCommit(r, skipMetadata)).collect(Collectors.toList());
        allCommits.addAll(readCommits);
        reader.close();
    }
    TableHeader header = new TableHeader().addTableHeaderField("CommitTime").addTableHeaderField("CommitType");
    if (!skipMetadata) {
        header = header.addTableHeaderField("CommitDetails");
    }
    return HoodiePrintHelper.print(header, new HashMap<>(), sortByField, descending, limit, headerOnly, allCommits);
}

Example 9 with ClosableIterator

use of org.apache.hudi.common.util.ClosableIterator in project hudi by apache.

the class MergeOnReadInputFormat method getLogFileIterator.

private ClosableIterator<RowData> getLogFileIterator(MergeOnReadInputSplit split) {
    final Schema tableSchema = new Schema.Parser().parse(tableState.getAvroSchema());
    final Schema requiredSchema = new Schema.Parser().parse(tableState.getRequiredAvroSchema());
    final GenericRecordBuilder recordBuilder = new GenericRecordBuilder(requiredSchema);
    final AvroToRowDataConverters.AvroToRowDataConverter avroToRowDataConverter = AvroToRowDataConverters.createRowConverter(tableState.getRequiredRowType());
    final HoodieMergedLogRecordScanner scanner = FormatUtils.logScanner(split, tableSchema, hadoopConf, conf.getBoolean(FlinkOptions.CHANGELOG_ENABLED));
    final Iterator<String> logRecordsKeyIterator = scanner.getRecords().keySet().iterator();
    final int[] pkOffset = tableState.getPkOffsetsInRequired();
    // flag saying whether the pk semantics has been dropped by user specified
    // projections. For e.g, if the pk fields are [a, b] but user only select a,
    // then the pk semantics is lost.
    final boolean pkSemanticLost = Arrays.stream(pkOffset).anyMatch(offset -> offset == -1);
    final LogicalType[] pkTypes = pkSemanticLost ? null : tableState.getPkTypes(pkOffset);
    final StringToRowDataConverter converter = pkSemanticLost ? null : new StringToRowDataConverter(pkTypes);
    return new ClosableIterator<RowData>() {

        private RowData currentRecord;

        @Override
        public boolean hasNext() {
            while (logRecordsKeyIterator.hasNext()) {
                String curAvroKey = logRecordsKeyIterator.next();
                Option<IndexedRecord> curAvroRecord = null;
                final HoodieAvroRecord<?> hoodieRecord = (HoodieAvroRecord) scanner.getRecords().get(curAvroKey);
                try {
                    curAvroRecord = hoodieRecord.getData().getInsertValue(tableSchema);
                } catch (IOException e) {
                    throw new HoodieException("Get avro insert value error for key: " + curAvroKey, e);
                }
                if (!curAvroRecord.isPresent()) {
                    // delete record found
                    if (emitDelete && !pkSemanticLost) {
                        GenericRowData delete = new GenericRowData(tableState.getRequiredRowType().getFieldCount());
                        final String recordKey = hoodieRecord.getRecordKey();
                        final String[] pkFields = KeyGenUtils.extractRecordKeys(recordKey);
                        final Object[] converted = converter.convert(pkFields);
                        for (int i = 0; i < pkOffset.length; i++) {
                            delete.setField(pkOffset[i], converted[i]);
                        }
                        delete.setRowKind(RowKind.DELETE);
                        this.currentRecord = delete;
                        return true;
                    }
                // skipping if the condition is unsatisfied
                // continue;
                } else {
                    final IndexedRecord avroRecord = curAvroRecord.get();
                    final RowKind rowKind = FormatUtils.getRowKindSafely(avroRecord, tableState.getOperationPos());
                    if (rowKind == RowKind.DELETE && !emitDelete) {
                        // skip the delete record
                        continue;
                    }
                    GenericRecord requiredAvroRecord = buildAvroRecordBySchema(avroRecord, requiredSchema, requiredPos, recordBuilder);
                    currentRecord = (RowData) avroToRowDataConverter.convert(requiredAvroRecord);
                    currentRecord.setRowKind(rowKind);
                    return true;
                }
            }
            return false;
        }

        @Override
        public RowData next() {
            return currentRecord;
        }

        @Override
        public void close() {
            scanner.close();
        }
    };
}

Example 10 with ClosableIterator

use of org.apache.hudi.common.util.ClosableIterator in project hudi by apache.

the class MergeOnReadInputFormat method getUnMergedLogFileIterator.

private ClosableIterator<RowData> getUnMergedLogFileIterator(MergeOnReadInputSplit split) {
    final Schema tableSchema = new Schema.Parser().parse(tableState.getAvroSchema());
    final Schema requiredSchema = new Schema.Parser().parse(tableState.getRequiredAvroSchema());
    final GenericRecordBuilder recordBuilder = new GenericRecordBuilder(requiredSchema);
    final AvroToRowDataConverters.AvroToRowDataConverter avroToRowDataConverter = AvroToRowDataConverters.createRowConverter(tableState.getRequiredRowType());
    final FormatUtils.BoundedMemoryRecords records = new FormatUtils.BoundedMemoryRecords(split, tableSchema, hadoopConf, conf);
    final Iterator<HoodieRecord<?>> recordsIterator = records.getRecordsIterator();
    return new ClosableIterator<RowData>() {

        private RowData currentRecord;

        @Override
        public boolean hasNext() {
            while (recordsIterator.hasNext()) {
                Option<IndexedRecord> curAvroRecord = null;
                final HoodieAvroRecord<?> hoodieRecord = (HoodieAvroRecord) recordsIterator.next();
                try {
                    curAvroRecord = hoodieRecord.getData().getInsertValue(tableSchema);
                } catch (IOException e) {
                    throw new HoodieException("Get avro insert value error for key: " + hoodieRecord.getRecordKey(), e);
                }
                if (curAvroRecord.isPresent()) {
                    final IndexedRecord avroRecord = curAvroRecord.get();
                    GenericRecord requiredAvroRecord = buildAvroRecordBySchema(avroRecord, requiredSchema, requiredPos, recordBuilder);
                    currentRecord = (RowData) avroToRowDataConverter.convert(requiredAvroRecord);
                    FormatUtils.setRowKind(currentRecord, avroRecord, tableState.getOperationPos());
                    return true;
                }
            }
            return false;
        }

        @Override
        public RowData next() {
            return currentRecord;
        }

        @Override
        public void close() {
            records.close();
        }
    };
}