Examples with BootstrapBaseFileMapping org.apache.hudi.common.model.BootstrapBaseFileMapping used on opensource projects

Example 1 with BootstrapBaseFileMapping

use of org.apache.hudi.common.model.BootstrapBaseFileMapping in project hudi by apache.

the class SpillableMapBasedFileSystemView method createFileIdToBootstrapBaseFileMap.

@Override
protected Map<HoodieFileGroupId, BootstrapBaseFileMapping> createFileIdToBootstrapBaseFileMap(Map<HoodieFileGroupId, BootstrapBaseFileMapping> fileGroupIdBootstrapBaseFileMap) {
    try {
        LOG.info("Creating bootstrap base File Map using external spillable Map. Max Mem=" + maxMemoryForBootstrapBaseFile + ", BaseDir=" + baseStoreDir);
        new File(baseStoreDir).mkdirs();
        Map<HoodieFileGroupId, BootstrapBaseFileMapping> pendingMap = new ExternalSpillableMap<>(maxMemoryForBootstrapBaseFile, baseStoreDir, new DefaultSizeEstimator(), new DefaultSizeEstimator<>(), diskMapType, isBitCaskDiskMapCompressionEnabled);
        pendingMap.putAll(fileGroupIdBootstrapBaseFileMap);
        return pendingMap;
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 2 with BootstrapBaseFileMapping

use of org.apache.hudi.common.model.BootstrapBaseFileMapping in project hudi by apache.

the class AbstractTableFileSystemView method addBootstrapBaseFileIfPresent.

protected HoodieBaseFile addBootstrapBaseFileIfPresent(HoodieFileGroupId fileGroupId, HoodieBaseFile baseFile) {
    if (baseFile.getCommitTime().equals(METADATA_BOOTSTRAP_INSTANT_TS)) {
        HoodieBaseFile copy = new HoodieBaseFile(baseFile);
        Option<BootstrapBaseFileMapping> edf = getBootstrapBaseFile(fileGroupId);
        edf.ifPresent(e -> copy.setBootstrapBaseFile(e.getBootstrapBaseFile()));
        return copy;
    }
    return baseFile;
}

Example 3 with BootstrapBaseFileMapping

use of org.apache.hudi.common.model.BootstrapBaseFileMapping in project hudi by apache.

the class AbstractTableFileSystemView method addFilesToView.

/**
 * Adds the provided statuses into the file system view, and also caches it inside this object.
 */
public List<HoodieFileGroup> addFilesToView(FileStatus[] statuses) {
    HoodieTimer timer = new HoodieTimer().startTimer();
    List<HoodieFileGroup> fileGroups = buildFileGroups(statuses, visibleCommitsAndCompactionTimeline, true);
    long fgBuildTimeTakenMs = timer.endTimer();
    timer.startTimer();
    // Group by partition for efficient updates for both InMemory and DiskBased stuctures.
    fileGroups.stream().collect(Collectors.groupingBy(HoodieFileGroup::getPartitionPath)).forEach((partition, value) -> {
        if (!isPartitionAvailableInStore(partition)) {
            if (bootstrapIndex.useIndex()) {
                try (BootstrapIndex.IndexReader reader = bootstrapIndex.createReader()) {
                    LOG.info("Bootstrap Index available for partition " + partition);
                    List<BootstrapFileMapping> sourceFileMappings = reader.getSourceFileMappingForPartition(partition);
                    addBootstrapBaseFileMapping(sourceFileMappings.stream().map(s -> new BootstrapBaseFileMapping(new HoodieFileGroupId(s.getPartitionPath(), s.getFileId()), s.getBootstrapFileStatus())));
                }
            }
            storePartitionView(partition, value);
        }
    });
    long storePartitionsTs = timer.endTimer();
    LOG.info("addFilesToView: NumFiles=" + statuses.length + ", NumFileGroups=" + fileGroups.size() + ", FileGroupsCreationTime=" + fgBuildTimeTakenMs + ", StoreTimeTaken=" + storePartitionsTs);
    return fileGroups;
}

Example 4 with BootstrapBaseFileMapping

use of org.apache.hudi.common.model.BootstrapBaseFileMapping in project hudi by apache.

the class RocksDbBasedFileSystemView method getBootstrapBaseFile.

@Override
protected Option<BootstrapBaseFileMapping> getBootstrapBaseFile(HoodieFileGroupId fileGroupId) {
    String lookupKey = schemaHelper.getKeyForBootstrapBaseFile(fileGroupId);
    BootstrapBaseFileMapping externalBaseFile = rocksDB.get(schemaHelper.getColFamilyForBootstrapBaseFile(), lookupKey);
    return Option.ofNullable(externalBaseFile);
}

Example 5 with BootstrapBaseFileMapping

use of org.apache.hudi.common.model.BootstrapBaseFileMapping in project hudi by apache.

the class AbstractTableFileSystemView method addBootstrapBaseFileIfPresent.

protected FileSlice addBootstrapBaseFileIfPresent(FileSlice fileSlice) {
    if (fileSlice.getBaseInstantTime().equals(METADATA_BOOTSTRAP_INSTANT_TS)) {
        FileSlice copy = new FileSlice(fileSlice);
        copy.getBaseFile().ifPresent(dataFile -> {
            Option<BootstrapBaseFileMapping> edf = getBootstrapBaseFile(copy.getFileGroupId());
            edf.ifPresent(e -> dataFile.setBootstrapBaseFile(e.getBootstrapBaseFile()));
        });
        return copy;
    }
    return fileSlice;
}