Examples with HoodiePayloadConfig org.apache.hudi.config.HoodiePayloadConfig used on opensource projects

Example 1 with HoodiePayloadConfig

use of org.apache.hudi.config.HoodiePayloadConfig in project hudi by apache.

the class HoodieFileSliceReader method getFileSliceReader.

public static HoodieFileSliceReader getFileSliceReader(Option<HoodieFileReader> baseFileReader, HoodieMergedLogRecordScanner scanner, Schema schema, String payloadClass, String preCombineField, Option<Pair<String, String>> simpleKeyGenFieldsOpt) throws IOException {
    if (baseFileReader.isPresent()) {
        Iterator baseIterator = baseFileReader.get().getRecordIterator(schema);
        while (baseIterator.hasNext()) {
            GenericRecord record = (GenericRecord) baseIterator.next();
            HoodieRecord<? extends HoodieRecordPayload> hoodieRecord = transform(record, scanner, payloadClass, preCombineField, simpleKeyGenFieldsOpt);
            scanner.processNextRecord(hoodieRecord);
        }
        return new HoodieFileSliceReader(scanner.iterator());
    } else {
        Iterable<HoodieRecord<? extends HoodieRecordPayload>> iterable = () -> scanner.iterator();
        HoodiePayloadConfig payloadConfig = HoodiePayloadConfig.newBuilder().withPayloadOrderingField(preCombineField).build();
        return new HoodieFileSliceReader(StreamSupport.stream(iterable.spliterator(), false).map(e -> {
            try {
                GenericRecord record = (GenericRecord) e.getData().getInsertValue(schema, payloadConfig.getProps()).get();
                return transform(record, scanner, payloadClass, preCombineField, simpleKeyGenFieldsOpt);
            } catch (IOException io) {
                throw new HoodieIOException("Error while creating reader for file slice with no base file.", io);
            }
        }).iterator());
    }
}