Examples with HoodieFileReader org.apache.hudi.io.storage.HoodieFileReader used on opensource projects

Example 11 with HoodieFileReader

use of org.apache.hudi.io.storage.HoodieFileReader in project hudi by apache.

the class FlinkMergeHelper method runMerge.

@Override
public void runMerge(HoodieTable<T, List<HoodieRecord<T>>, List<HoodieKey>, List<WriteStatus>> table, HoodieMergeHandle<T, List<HoodieRecord<T>>, List<HoodieKey>, List<WriteStatus>> mergeHandle) throws IOException {
    final GenericDatumWriter<GenericRecord> gWriter;
    final GenericDatumReader<GenericRecord> gReader;
    Schema readSchema;
    final boolean externalSchemaTransformation = table.getConfig().shouldUseExternalSchemaTransformation();
    HoodieBaseFile baseFile = mergeHandle.baseFileForMerge();
    if (externalSchemaTransformation || baseFile.getBootstrapBaseFile().isPresent()) {
        readSchema = HoodieFileReaderFactory.getFileReader(table.getHadoopConf(), mergeHandle.getOldFilePath()).getSchema();
        gWriter = new GenericDatumWriter<>(readSchema);
        gReader = new GenericDatumReader<>(readSchema, mergeHandle.getWriterSchemaWithMetaFields());
    } else {
        gReader = null;
        gWriter = null;
        readSchema = mergeHandle.getWriterSchemaWithMetaFields();
    }
    BoundedInMemoryExecutor<GenericRecord, GenericRecord, Void> wrapper = null;
    Configuration cfgForHoodieFile = new Configuration(table.getHadoopConf());
    HoodieFileReader<GenericRecord> reader = HoodieFileReaderFactory.<GenericRecord>getFileReader(cfgForHoodieFile, mergeHandle.getOldFilePath());
    try {
        final Iterator<GenericRecord> readerIterator;
        if (baseFile.getBootstrapBaseFile().isPresent()) {
            readerIterator = getMergingIterator(table, mergeHandle, baseFile, reader, readSchema, externalSchemaTransformation);
        } else {
            readerIterator = reader.getRecordIterator(readSchema);
        }
        ThreadLocal<BinaryEncoder> encoderCache = new ThreadLocal<>();
        ThreadLocal<BinaryDecoder> decoderCache = new ThreadLocal<>();
        wrapper = new BoundedInMemoryExecutor<>(table.getConfig().getWriteBufferLimitBytes(), new IteratorBasedQueueProducer<>(readerIterator), Option.of(new UpdateHandler(mergeHandle)), record -> {
            if (!externalSchemaTransformation) {
                return record;
            }
            return transformRecordBasedOnNewSchema(gReader, gWriter, encoderCache, decoderCache, (GenericRecord) record);
        });
        wrapper.execute();
    } catch (Exception e) {
        throw new HoodieException(e);
    } finally {
        if (reader != null) {
            reader.close();
        }
        mergeHandle.close();
        if (null != wrapper) {
            wrapper.shutdownNow();
        }
    }
}

Example 12 with HoodieFileReader

use of org.apache.hudi.io.storage.HoodieFileReader in project hudi by apache.

the class HoodieBackedTableMetadata method getBaseFileReader.

private Pair<HoodieFileReader, Long> getBaseFileReader(FileSlice slice, HoodieTimer timer) throws IOException {
    HoodieFileReader baseFileReader = null;
    Long baseFileOpenMs;
    // If the base file is present then create a reader
    Option<HoodieBaseFile> basefile = slice.getBaseFile();
    if (basefile.isPresent()) {
        String basefilePath = basefile.get().getPath();
        baseFileReader = HoodieFileReaderFactory.getFileReader(hadoopConf.get(), new Path(basefilePath));
        baseFileOpenMs = timer.endTimer();
        LOG.info(String.format("Opened metadata base file from %s at instant %s in %d ms", basefilePath, basefile.get().getCommitTime(), baseFileOpenMs));
    } else {
        baseFileOpenMs = 0L;
        timer.endTimer();
    }
    return Pair.of(baseFileReader, baseFileOpenMs);
}

Example 13 with HoodieFileReader

use of org.apache.hudi.io.storage.HoodieFileReader in project hudi by apache.

the class HoodieBackedTableMetadata method getRecordsByKeys.

@Override
protected List<Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>>> getRecordsByKeys(List<String> keys, String partitionName) {
    Map<Pair<String, FileSlice>, List<String>> partitionFileSliceToKeysMap = getPartitionFileSliceToKeysMapping(partitionName, keys);
    List<Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>>> result = new ArrayList<>();
    AtomicInteger fileSlicesKeysCount = new AtomicInteger();
    partitionFileSliceToKeysMap.forEach((partitionFileSlicePair, fileSliceKeys) -> {
        Pair<HoodieFileReader, HoodieMetadataMergedLogRecordReader> readers = openReadersIfNeeded(partitionName, partitionFileSlicePair.getRight());
        try {
            List<Long> timings = new ArrayList<>();
            HoodieFileReader baseFileReader = readers.getKey();
            HoodieMetadataMergedLogRecordReader logRecordScanner = readers.getRight();
            if (baseFileReader == null && logRecordScanner == null) {
                return;
            }
            // local map to assist in merging with base file records
            Map<String, Option<HoodieRecord<HoodieMetadataPayload>>> logRecords = readLogRecords(logRecordScanner, fileSliceKeys, timings);
            result.addAll(readFromBaseAndMergeWithLogRecords(baseFileReader, fileSliceKeys, logRecords, timings, partitionName));
            LOG.debug(String.format("Metadata read for %s keys took [baseFileRead, logMerge] %s ms", fileSliceKeys.size(), timings));
            fileSlicesKeysCount.addAndGet(fileSliceKeys.size());
        } catch (IOException ioe) {
            throw new HoodieIOException("Error merging records from metadata table for  " + keys.size() + " key : ", ioe);
        } finally {
            if (!reuse) {
                close(Pair.of(partitionFileSlicePair.getLeft(), partitionFileSlicePair.getRight().getFileId()));
            }
        }
    });
    ValidationUtils.checkState(keys.size() == fileSlicesKeysCount.get());
    return result;
}

Example 14 with HoodieFileReader

use of org.apache.hudi.io.storage.HoodieFileReader in project hudi by apache.

the class HoodieBackedTableMetadata method openReadersIfNeeded.

/**
 * Create a file reader and the record scanner for a given partition and file slice
 * if readers are not already available.
 *
 * @param partitionName - Partition name
 * @param slice         - The file slice to open readers for
 * @return File reader and the record scanner pair for the requested file slice
 */
private Pair<HoodieFileReader, HoodieMetadataMergedLogRecordReader> openReadersIfNeeded(String partitionName, FileSlice slice) {
    return partitionReaders.computeIfAbsent(Pair.of(partitionName, slice.getFileId()), k -> {
        try {
            HoodieTimer timer = new HoodieTimer().startTimer();
            // Open base file reader
            Pair<HoodieFileReader, Long> baseFileReaderOpenTimePair = getBaseFileReader(slice, timer);
            HoodieFileReader baseFileReader = baseFileReaderOpenTimePair.getKey();
            final long baseFileOpenMs = baseFileReaderOpenTimePair.getValue();
            // Open the log record scanner using the log files from the latest file slice
            List<HoodieLogFile> logFiles = slice.getLogFiles().collect(Collectors.toList());
            Pair<HoodieMetadataMergedLogRecordReader, Long> logRecordScannerOpenTimePair = getLogRecordScanner(logFiles, partitionName);
            HoodieMetadataMergedLogRecordReader logRecordScanner = logRecordScannerOpenTimePair.getKey();
            final long logScannerOpenMs = logRecordScannerOpenTimePair.getValue();
            metrics.ifPresent(metrics -> metrics.updateMetrics(HoodieMetadataMetrics.SCAN_STR, +baseFileOpenMs + logScannerOpenMs));
            return Pair.of(baseFileReader, logRecordScanner);
        } catch (IOException e) {
            throw new HoodieIOException("Error opening readers for metadata table partition " + partitionName, e);
        }
    });
}

Example 15 with HoodieFileReader

use of org.apache.hudi.io.storage.HoodieFileReader in project hudi by apache.

the class MultipleSparkJobExecutionStrategy method readRecordsForGroupBaseFiles.

/**
 * Read records from baseFiles and convert to RDD.
 */
private HoodieData<HoodieRecord<T>> readRecordsForGroupBaseFiles(JavaSparkContext jsc, List<ClusteringOperation> clusteringOps) {
    SerializableConfiguration hadoopConf = new SerializableConfiguration(getHoodieTable().getHadoopConf());
    HoodieWriteConfig writeConfig = getWriteConfig();
    // closure, as this might lead to issues attempting to serialize its nested fields
    return HoodieJavaRDD.of(jsc.parallelize(clusteringOps, clusteringOps.size()).mapPartitions(clusteringOpsPartition -> {
        List<Iterator<IndexedRecord>> iteratorsForPartition = new ArrayList<>();
        clusteringOpsPartition.forEachRemaining(clusteringOp -> {
            try {
                Schema readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(writeConfig.getSchema()));
                HoodieFileReader<IndexedRecord> baseFileReader = HoodieFileReaderFactory.getFileReader(hadoopConf.get(), new Path(clusteringOp.getDataFilePath()));
                iteratorsForPartition.add(baseFileReader.getRecordIterator(readerSchema));
            } catch (IOException e) {
                throw new HoodieClusteringException("Error reading input data for " + clusteringOp.getDataFilePath() + " and " + clusteringOp.getDeltaFilePaths(), e);
            }
        });
        return new ConcatenatingIterator<>(iteratorsForPartition);
    }).map(record -> transform(record, writeConfig)));
}