Search in sources :

Example 26 with HoodieTimer

use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.

the class BaseTableMetadata method getColumnStats.

@Override
public Map<Pair<String, String>, HoodieMetadataColumnStats> getColumnStats(final List<Pair<String, String>> partitionNameFileNameList, final String columnName) throws HoodieMetadataException {
    if (!isColumnStatsIndexEnabled) {
        LOG.error("Metadata column stats index is disabled!");
        return Collections.emptyMap();
    }
    Map<String, Pair<String, String>> columnStatKeyToFileNameMap = new HashMap<>();
    TreeSet<String> sortedKeys = new TreeSet<>();
    final ColumnIndexID columnIndexID = new ColumnIndexID(columnName);
    for (Pair<String, String> partitionNameFileNamePair : partitionNameFileNameList) {
        final String columnStatsIndexKey = HoodieMetadataPayload.getColumnStatsIndexKey(new PartitionIndexID(partitionNameFileNamePair.getLeft()), new FileIndexID(partitionNameFileNamePair.getRight()), columnIndexID);
        sortedKeys.add(columnStatsIndexKey);
        columnStatKeyToFileNameMap.put(columnStatsIndexKey, partitionNameFileNamePair);
    }
    List<String> columnStatKeys = new ArrayList<>(sortedKeys);
    HoodieTimer timer = new HoodieTimer().startTimer();
    List<Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>>> hoodieRecordList = getRecordsByKeys(columnStatKeys, MetadataPartitionType.COLUMN_STATS.getPartitionPath());
    metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_COLUMN_STATS_METADATA_STR, timer.endTimer()));
    Map<Pair<String, String>, HoodieMetadataColumnStats> fileToColumnStatMap = new HashMap<>();
    for (final Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>> entry : hoodieRecordList) {
        if (entry.getRight().isPresent()) {
            final Option<HoodieMetadataColumnStats> columnStatMetadata = entry.getRight().get().getData().getColumnStatMetadata();
            if (columnStatMetadata.isPresent()) {
                if (!columnStatMetadata.get().getIsDeleted()) {
                    ValidationUtils.checkState(columnStatKeyToFileNameMap.containsKey(entry.getLeft()));
                    final Pair<String, String> partitionFileNamePair = columnStatKeyToFileNameMap.get(entry.getLeft());
                    ValidationUtils.checkState(!fileToColumnStatMap.containsKey(partitionFileNamePair));
                    fileToColumnStatMap.put(partitionFileNamePair, columnStatMetadata.get());
                }
            } else {
                LOG.error("Meta index column stats missing for: " + entry.getLeft());
            }
        }
    }
    return fileToColumnStatMap;
}
Also used : HashMap(java.util.HashMap) HoodieRecord(org.apache.hudi.common.model.HoodieRecord) ArrayList(java.util.ArrayList) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) HoodieMetadataColumnStats(org.apache.hudi.avro.model.HoodieMetadataColumnStats) PartitionIndexID(org.apache.hudi.common.util.hash.PartitionIndexID) TreeSet(java.util.TreeSet) ColumnIndexID(org.apache.hudi.common.util.hash.ColumnIndexID) Option(org.apache.hudi.common.util.Option) FileIndexID(org.apache.hudi.common.util.hash.FileIndexID) Pair(org.apache.hudi.common.util.collection.Pair)

Example 27 with HoodieTimer

use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.

the class BaseTableMetadata method getBloomFilters.

@Override
public Map<Pair<String, String>, BloomFilter> getBloomFilters(final List<Pair<String, String>> partitionNameFileNameList) throws HoodieMetadataException {
    if (!isBloomFilterIndexEnabled) {
        LOG.error("Metadata bloom filter index is disabled!");
        return Collections.emptyMap();
    }
    if (partitionNameFileNameList.isEmpty()) {
        return Collections.emptyMap();
    }
    HoodieTimer timer = new HoodieTimer().startTimer();
    Set<String> partitionIDFileIDSortedStrings = new TreeSet<>();
    Map<String, Pair<String, String>> fileToKeyMap = new HashMap<>();
    partitionNameFileNameList.forEach(partitionNameFileNamePair -> {
        final String bloomFilterIndexKey = HoodieMetadataPayload.getBloomFilterIndexKey(new PartitionIndexID(partitionNameFileNamePair.getLeft()), new FileIndexID(partitionNameFileNamePair.getRight()));
        partitionIDFileIDSortedStrings.add(bloomFilterIndexKey);
        fileToKeyMap.put(bloomFilterIndexKey, partitionNameFileNamePair);
    });
    List<String> partitionIDFileIDStrings = new ArrayList<>(partitionIDFileIDSortedStrings);
    List<Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>>> hoodieRecordList = getRecordsByKeys(partitionIDFileIDStrings, MetadataPartitionType.BLOOM_FILTERS.getPartitionPath());
    metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_BLOOM_FILTERS_METADATA_STR, (timer.endTimer() / partitionIDFileIDStrings.size())));
    Map<Pair<String, String>, BloomFilter> partitionFileToBloomFilterMap = new HashMap<>();
    for (final Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>> entry : hoodieRecordList) {
        if (entry.getRight().isPresent()) {
            final Option<HoodieMetadataBloomFilter> bloomFilterMetadata = entry.getRight().get().getData().getBloomFilterMetadata();
            if (bloomFilterMetadata.isPresent()) {
                if (!bloomFilterMetadata.get().getIsDeleted()) {
                    ValidationUtils.checkState(fileToKeyMap.containsKey(entry.getLeft()));
                    final ByteBuffer bloomFilterByteBuffer = bloomFilterMetadata.get().getBloomFilter();
                    final String bloomFilterType = bloomFilterMetadata.get().getType();
                    final BloomFilter bloomFilter = BloomFilterFactory.fromString(StandardCharsets.UTF_8.decode(bloomFilterByteBuffer).toString(), bloomFilterType);
                    partitionFileToBloomFilterMap.put(fileToKeyMap.get(entry.getLeft()), bloomFilter);
                }
            } else {
                LOG.error("Meta index bloom filter missing for: " + fileToKeyMap.get(entry.getLeft()));
            }
        }
    }
    return partitionFileToBloomFilterMap;
}
Also used : HashMap(java.util.HashMap) HoodieRecord(org.apache.hudi.common.model.HoodieRecord) ArrayList(java.util.ArrayList) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) ByteBuffer(java.nio.ByteBuffer) HoodieMetadataBloomFilter(org.apache.hudi.avro.model.HoodieMetadataBloomFilter) BloomFilter(org.apache.hudi.common.bloom.BloomFilter) PartitionIndexID(org.apache.hudi.common.util.hash.PartitionIndexID) HoodieMetadataBloomFilter(org.apache.hudi.avro.model.HoodieMetadataBloomFilter) TreeSet(java.util.TreeSet) Option(org.apache.hudi.common.util.Option) FileIndexID(org.apache.hudi.common.util.hash.FileIndexID) Pair(org.apache.hudi.common.util.collection.Pair)

Example 28 with HoodieTimer

use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.

the class BaseTableMetadata method fetchAllFilesInPartitionPaths.

Map<String, FileStatus[]> fetchAllFilesInPartitionPaths(List<Path> partitionPaths) throws IOException {
    Map<String, Path> partitionInfo = new HashMap<>();
    boolean foundNonPartitionedPath = false;
    for (Path partitionPath : partitionPaths) {
        String partitionName = FSUtils.getRelativePartitionPath(new Path(dataBasePath), partitionPath);
        if (partitionName.isEmpty()) {
            if (partitionInfo.size() > 1) {
                throw new HoodieMetadataException("Found mix of partitioned and non partitioned paths while fetching data from metadata table");
            }
            partitionInfo.put(NON_PARTITIONED_NAME, partitionPath);
            foundNonPartitionedPath = true;
        } else {
            if (foundNonPartitionedPath) {
                throw new HoodieMetadataException("Found mix of partitioned and non partitioned paths while fetching data from metadata table");
            }
            partitionInfo.put(partitionName, partitionPath);
        }
    }
    HoodieTimer timer = new HoodieTimer().startTimer();
    List<Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>>> partitionsFileStatus = getRecordsByKeys(new ArrayList<>(partitionInfo.keySet()), MetadataPartitionType.FILES.getPartitionPath());
    metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_FILES_STR, timer.endTimer()));
    Map<String, FileStatus[]> result = new HashMap<>();
    for (Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>> entry : partitionsFileStatus) {
        if (entry.getValue().isPresent()) {
            mayBeHandleSpuriousDeletes(entry.getValue(), entry.getKey());
            result.put(partitionInfo.get(entry.getKey()).toString(), entry.getValue().get().getData().getFileStatuses(hadoopConf.get(), partitionInfo.get(entry.getKey())));
        }
    }
    LOG.info("Listed files in partitions from metadata: partition list =" + Arrays.toString(partitionPaths.toArray()));
    return result;
}
Also used : Path(org.apache.hadoop.fs.Path) HashMap(java.util.HashMap) HoodieRecord(org.apache.hudi.common.model.HoodieRecord) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) HoodieMetadataException(org.apache.hudi.exception.HoodieMetadataException) Option(org.apache.hudi.common.util.Option) Pair(org.apache.hudi.common.util.collection.Pair)

Example 29 with HoodieTimer

use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.

the class HoodieBackedTableMetadata method getLogRecordScanner.

public Pair<HoodieMetadataMergedLogRecordReader, Long> getLogRecordScanner(List<HoodieLogFile> logFiles, String partitionName) {
    HoodieTimer timer = new HoodieTimer().startTimer();
    List<String> sortedLogFilePaths = logFiles.stream().sorted(HoodieLogFile.getLogFileComparator()).map(o -> o.getPath().toString()).collect(Collectors.toList());
    // Only those log files which have a corresponding completed instant on the dataset should be read
    // This is because the metadata table is updated before the dataset instants are committed.
    Set<String> validInstantTimestamps = getValidInstantTimestamps();
    Option<HoodieInstant> latestMetadataInstant = metadataMetaClient.getActiveTimeline().filterCompletedInstants().lastInstant();
    String latestMetadataInstantTime = latestMetadataInstant.map(HoodieInstant::getTimestamp).orElse(SOLO_COMMIT_TIMESTAMP);
    // Load the schema
    Schema schema = HoodieAvroUtils.addMetadataFields(HoodieMetadataRecord.getClassSchema());
    HoodieCommonConfig commonConfig = HoodieCommonConfig.newBuilder().fromProperties(metadataConfig.getProps()).build();
    HoodieMetadataMergedLogRecordReader logRecordScanner = HoodieMetadataMergedLogRecordReader.newBuilder().withFileSystem(metadataMetaClient.getFs()).withBasePath(metadataBasePath).withLogFilePaths(sortedLogFilePaths).withReaderSchema(schema).withLatestInstantTime(latestMetadataInstantTime).withMaxMemorySizeInBytes(MAX_MEMORY_SIZE_IN_BYTES).withBufferSize(BUFFER_SIZE).withSpillableMapBasePath(spillableMapDirectory).withDiskMapType(commonConfig.getSpillableDiskMapType()).withBitCaskDiskMapCompressionEnabled(commonConfig.isBitCaskDiskMapCompressionEnabled()).withLogBlockTimestamps(validInstantTimestamps).enableFullScan(metadataConfig.enableFullScan()).withPartition(partitionName).build();
    Long logScannerOpenMs = timer.endTimer();
    LOG.info(String.format("Opened %d metadata log files (dataset instant=%s, metadata instant=%s) in %d ms", sortedLogFilePaths.size(), getLatestDataInstantTime(), latestMetadataInstantTime, logScannerOpenMs));
    return Pair.of(logRecordScanner, logScannerOpenMs);
}
Also used : HoodieAvroUtils(org.apache.hudi.avro.HoodieAvroUtils) HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) FileSlice(org.apache.hudi.common.model.FileSlice) HoodieException(org.apache.hudi.exception.HoodieException) Option(org.apache.hudi.common.util.Option) HashMap(java.util.HashMap) HoodieEngineContext(org.apache.hudi.common.engine.HoodieEngineContext) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) ArrayList(java.util.ArrayList) Logger(org.apache.log4j.Logger) HoodieFileReaderFactory(org.apache.hudi.io.storage.HoodieFileReaderFactory) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) HoodieTableConfig(org.apache.hudi.common.table.HoodieTableConfig) HoodieTableMetaClient(org.apache.hudi.common.table.HoodieTableMetaClient) Map(java.util.Map) HoodieRollbackMetadata(org.apache.hudi.avro.model.HoodieRollbackMetadata) Path(org.apache.hadoop.fs.Path) HoodieLogFile(org.apache.hudi.common.model.HoodieLogFile) LinkedList(java.util.LinkedList) HoodieActiveTimeline(org.apache.hudi.common.table.timeline.HoodieActiveTimeline) HoodieTimeline(org.apache.hudi.common.table.timeline.HoodieTimeline) HoodieMetadataConfig(org.apache.hudi.common.config.HoodieMetadataConfig) HoodieRecord(org.apache.hudi.common.model.HoodieRecord) ValidationUtils(org.apache.hudi.common.util.ValidationUtils) TableNotFoundException(org.apache.hudi.exception.TableNotFoundException) GenericRecord(org.apache.avro.generic.GenericRecord) Schema(org.apache.avro.Schema) HoodieFileReader(org.apache.hudi.io.storage.HoodieFileReader) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) TimelineMetadataUtils(org.apache.hudi.common.table.timeline.TimelineMetadataUtils) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) HoodieAvroRecord(org.apache.hudi.common.model.HoodieAvroRecord) HoodieMetadataException(org.apache.hudi.exception.HoodieMetadataException) HoodieRecordPayload(org.apache.hudi.common.model.HoodieRecordPayload) HoodieBaseFile(org.apache.hudi.common.model.HoodieBaseFile) List(java.util.List) HoodieMetadataRecord(org.apache.hudi.avro.model.HoodieMetadataRecord) SerializableConfiguration(org.apache.hudi.common.config.SerializableConfiguration) HoodieRestoreMetadata(org.apache.hudi.avro.model.HoodieRestoreMetadata) HoodieIOException(org.apache.hudi.exception.HoodieIOException) LogManager(org.apache.log4j.LogManager) HoodieCommonConfig(org.apache.hudi.common.config.HoodieCommonConfig) SpillableMapUtils(org.apache.hudi.common.util.SpillableMapUtils) Collections(java.util.Collections) Pair(org.apache.hudi.common.util.collection.Pair) HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) HoodieCommonConfig(org.apache.hudi.common.config.HoodieCommonConfig) Schema(org.apache.avro.Schema) HoodieTimer(org.apache.hudi.common.util.HoodieTimer)

Example 30 with HoodieTimer

use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.

the class HoodieBackedTableMetadata method readLogRecords.

private Map<String, Option<HoodieRecord<HoodieMetadataPayload>>> readLogRecords(HoodieMetadataMergedLogRecordReader logRecordScanner, List<String> keys, List<Long> timings) {
    HoodieTimer timer = new HoodieTimer().startTimer();
    Map<String, Option<HoodieRecord<HoodieMetadataPayload>>> logRecords = new HashMap<>();
    // Retrieve records from log file
    timer.startTimer();
    if (logRecordScanner != null) {
        if (metadataConfig.enableFullScan()) {
            // path which does full scan of log files
            for (String key : keys) {
                logRecords.put(key, logRecordScanner.getRecordByKey(key).get(0).getValue());
            }
        } else {
            // this path will do seeks pertaining to the keys passed in
            List<Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>>> logRecordsList = logRecordScanner.getRecordsByKeys(keys);
            for (Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>> entry : logRecordsList) {
                logRecords.put(entry.getKey(), entry.getValue());
            }
        }
    } else {
        for (String key : keys) {
            logRecords.put(key, Option.empty());
        }
    }
    timings.add(timer.endTimer());
    return logRecords;
}
Also used : HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HoodieRecord(org.apache.hudi.common.model.HoodieRecord) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) Option(org.apache.hudi.common.util.Option) Pair(org.apache.hudi.common.util.collection.Pair)

Aggregations

HoodieTimer (org.apache.hudi.common.util.HoodieTimer)35 ArrayList (java.util.ArrayList)16 Path (org.apache.hadoop.fs.Path)15 IOException (java.io.IOException)14 HashMap (java.util.HashMap)12 Option (org.apache.hudi.common.util.Option)12 HoodieInstant (org.apache.hudi.common.table.timeline.HoodieInstant)11 HoodieRecord (org.apache.hudi.common.model.HoodieRecord)10 Map (java.util.Map)9 Pair (org.apache.hudi.common.util.collection.Pair)9 List (java.util.List)8 FileStatus (org.apache.hadoop.fs.FileStatus)8 HoodieIOException (org.apache.hudi.exception.HoodieIOException)7 LogManager (org.apache.log4j.LogManager)7 Logger (org.apache.log4j.Logger)7 Collectors (java.util.stream.Collectors)6 HoodieSparkEngineContext (org.apache.hudi.client.common.HoodieSparkEngineContext)6 HoodieTableMetaClient (org.apache.hudi.common.table.HoodieTableMetaClient)6 HoodieTimeline (org.apache.hudi.common.table.timeline.HoodieTimeline)6 HoodieWriteConfig (org.apache.hudi.config.HoodieWriteConfig)6