Examples with StoreFileInfo org.apache.hadoop.hbase.regionserver.StoreFileInfo used on opensource projects

Example 1 with StoreFileInfo

use of org.apache.hadoop.hbase.regionserver.StoreFileInfo in project hbase by apache.

the class SnapshotManifest method addMobRegion.

public void addMobRegion(HRegionInfo regionInfo) throws IOException {
    // 0. Get the ManifestBuilder/RegionVisitor
    RegionVisitor visitor = createRegionVisitor(desc);
    // 1. dump region meta info into the snapshot directory
    LOG.debug("Storing mob region '" + regionInfo + "' region-info for snapshot.");
    Object regionData = visitor.regionOpen(regionInfo);
    monitor.rethrowException();
    // 2. iterate through all the stores in the region
    LOG.debug("Creating references for mob files");
    Path mobRegionPath = MobUtils.getMobRegionPath(conf, regionInfo.getTable());
    for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
        // 2.1. build the snapshot reference for the store if it's a mob store
        if (!hcd.isMobEnabled()) {
            continue;
        }
        Object familyData = visitor.familyOpen(regionData, hcd.getName());
        monitor.rethrowException();
        Path storePath = MobUtils.getMobFamilyPath(mobRegionPath, hcd.getNameAsString());
        List<StoreFileInfo> storeFiles = getStoreFiles(storePath);
        if (storeFiles == null) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("No mob files under family: " + hcd.getNameAsString());
            }
            continue;
        }
        addReferenceFiles(visitor, regionData, familyData, storeFiles, true);
        visitor.familyClose(regionData, familyData);
    }
    visitor.regionClose(regionData);
}

Example 2 with StoreFileInfo

use of org.apache.hadoop.hbase.regionserver.StoreFileInfo in project hbase by apache.

the class MergeTableRegionsProcedure method mergeStoreFiles.

/**
   * Create reference file(s) of merging regions under the merges directory
   * @param env MasterProcedureEnv
   * @param regionFs region file system
   * @param mergedDir the temp directory of merged region
   * @throws IOException
   */
private void mergeStoreFiles(final MasterProcedureEnv env, final HRegionFileSystem regionFs, final Path mergedDir) throws IOException {
    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
    final Configuration conf = env.getMasterConfiguration();
    final HTableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName());
    for (String family : regionFs.getFamilies()) {
        final HColumnDescriptor hcd = htd.getFamily(family.getBytes());
        final Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(family);
        if (storeFiles != null && storeFiles.size() > 0) {
            final CacheConfig cacheConf = new CacheConfig(conf, hcd);
            for (StoreFileInfo storeFileInfo : storeFiles) {
                // Create reference file(s) of the region in mergedDir
                regionFs.mergeStoreFile(mergedRegionInfo, family, new StoreFile(mfs.getFileSystem(), storeFileInfo, conf, cacheConf, hcd.getBloomFilterType()), mergedDir);
            }
        }
    }
}

Example 3 with StoreFileInfo

use of org.apache.hadoop.hbase.regionserver.StoreFileInfo in project hbase by apache.

the class MasterRegion method tryMigrate.

private static void tryMigrate(Configuration conf, FileSystem fs, Path tableDir, RegionInfo regionInfo, TableDescriptor oldTd, TableDescriptor newTd) throws IOException {
    Class<? extends StoreFileTracker> oldSft = StoreFileTrackerFactory.getTrackerClass(oldTd.getValue(StoreFileTrackerFactory.TRACKER_IMPL));
    Class<? extends StoreFileTracker> newSft = StoreFileTrackerFactory.getTrackerClass(newTd.getValue(StoreFileTrackerFactory.TRACKER_IMPL));
    if (oldSft.equals(newSft)) {
        LOG.debug("old store file tracker {} is the same with new store file tracker, skip migration", StoreFileTrackerFactory.getStoreFileTrackerName(oldSft));
        if (!oldTd.equals(newTd)) {
            // we may change other things such as adding a new family, so here we still need to persist
            // the new table descriptor
            LOG.info("Update table descriptor from {} to {}", oldTd, newTd);
            FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, newTd, true);
        }
        return;
    }
    LOG.info("Migrate store file tracker from {} to {}", oldSft.getSimpleName(), newSft.getSimpleName());
    HRegionFileSystem hfs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, false);
    for (ColumnFamilyDescriptor oldCfd : oldTd.getColumnFamilies()) {
        StoreFileTracker oldTracker = StoreFileTrackerFactory.create(conf, oldTd, oldCfd, hfs);
        StoreFileTracker newTracker = StoreFileTrackerFactory.create(conf, oldTd, oldCfd, hfs);
        List<StoreFileInfo> files = oldTracker.load();
        LOG.debug("Store file list for {}: {}", oldCfd.getNameAsString(), files);
        newTracker.set(oldTracker.load());
    }
    // persist the new table descriptor after migration
    LOG.info("Update table descriptor from {} to {}", oldTd, newTd);
    FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, newTd, true);
}

Example 4 with StoreFileInfo

use of org.apache.hadoop.hbase.regionserver.StoreFileInfo in project hbase by apache.

the class SnapshotManifest method addRegion.

protected void addRegion(Path tableDir, RegionInfo regionInfo, RegionVisitor visitor) throws IOException {
    boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo);
    try {
        Path baseDir = tableDir;
        // Open the RegionFS
        if (isMobRegion) {
            baseDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), regionInfo.getTable());
        }
        HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, rootFs, baseDir, regionInfo, true);
        monitor.rethrowException();
        // 1. dump region meta info into the snapshot directory
        LOG.debug("Storing region-info for snapshot.");
        Object regionData = visitor.regionOpen(regionInfo);
        monitor.rethrowException();
        // 2. iterate through all the stores in the region
        LOG.debug("Creating references for hfiles");
        // 1000 files/batch, far more than the number of store files under a single column family.
        for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) {
            Object familyData = visitor.familyOpen(regionData, cfd.getName());
            monitor.rethrowException();
            StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, htd, cfd, regionFs);
            List<StoreFileInfo> storeFiles = tracker.load();
            if (storeFiles.isEmpty()) {
                LOG.debug("No files under family: {}", cfd.getNameAsString());
                continue;
            }
            // 2.1. build the snapshot reference for the store
            // iterate through all the store's files and create "references".
            addReferenceFiles(visitor, regionData, familyData, storeFiles, false);
            visitor.familyClose(regionData, familyData);
        }
        visitor.regionClose(regionData);
    } catch (IOException e) {
        // the mob directory might not be created yet, so do nothing when it is a mob region
        if (!isMobRegion) {
            throw e;
        }
    }
}

Example 5 with StoreFileInfo

use of org.apache.hadoop.hbase.regionserver.StoreFileInfo in project hbase by apache.

the class RestoreSnapshotHelper method restoreRegion.

/**
 * Restore region by removing files not in the snapshot
 * and adding the missing ones from the snapshot.
 */
private void restoreRegion(final RegionInfo regionInfo, final SnapshotRegionManifest regionManifest, Path regionDir) throws IOException {
    Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles = getRegionHFileReferences(regionManifest);
    String tableName = tableDesc.getTableName().getNameAsString();
    final String snapshotName = snapshotDesc.getName();
    Path regionPath = new Path(tableDir, regionInfo.getEncodedName());
    HRegionFileSystem regionFS = (fs.exists(regionPath)) ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, false) : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, regionInfo);
    // Restore families present in the table
    for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) {
        byte[] family = Bytes.toBytes(familyDir.getName());
        Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
        List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName());
        List<StoreFileInfo> filesToTrack = new ArrayList<>();
        if (snapshotFamilyFiles != null) {
            List<SnapshotRegionManifest.StoreFile> hfilesToAdd = new ArrayList<>();
            for (SnapshotRegionManifest.StoreFile storeFile : snapshotFamilyFiles) {
                if (familyFiles.contains(storeFile.getName())) {
                    // HFile already present
                    familyFiles.remove(storeFile.getName());
                    // no need to restore already present files, but we need to add those to tracker
                    filesToTrack.add(new StoreFileInfo(conf, fs, new Path(familyDir, storeFile.getName()), true));
                } else {
                    // HFile missing
                    hfilesToAdd.add(storeFile);
                }
            }
            // Remove hfiles not present in the snapshot
            for (String hfileName : familyFiles) {
                Path hfile = new Path(familyDir, hfileName);
                if (!fs.getFileStatus(hfile).isDirectory()) {
                    LOG.trace("Removing HFile=" + hfileName + " not present in snapshot=" + snapshotName + " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
                    HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
                }
            }
            // Restore Missing files
            for (SnapshotRegionManifest.StoreFile storeFile : hfilesToAdd) {
                LOG.debug("Restoring missing HFileLink " + storeFile.getName() + " of snapshot=" + snapshotName + " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
                String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
                // mark the reference file to be added to tracker
                filesToTrack.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true));
            }
        } else {
            // Family doesn't exists in the snapshot
            LOG.trace("Removing family=" + Bytes.toString(family) + " in snapshot=" + snapshotName + " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
            HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family);
            fs.delete(familyDir, true);
        }
        StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
        // simply reset list of tracked files with the matching files
        // and the extra one present in the snapshot
        tracker.set(filesToTrack);
    }
    // Add families not present in the table
    for (Map.Entry<String, List<SnapshotRegionManifest.StoreFile>> familyEntry : snapshotFiles.entrySet()) {
        Path familyDir = new Path(regionDir, familyEntry.getKey());
        StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
        List<StoreFileInfo> files = new ArrayList<>();
        if (!fs.mkdirs(familyDir)) {
            throw new IOException("Unable to create familyDir=" + familyDir);
        }
        for (SnapshotRegionManifest.StoreFile storeFile : familyEntry.getValue()) {
            LOG.trace("Adding HFileLink (Not present in the table) " + storeFile.getName() + " of snapshot " + snapshotName + " to table=" + tableName);
            String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
            files.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true));
        }
        tracker.set(files);
    }
}