Examples with HRegionFileSystem org.apache.hadoop.hbase.regionserver.HRegionFileSystem used on opensource projects

Example 16 with HRegionFileSystem

use of org.apache.hadoop.hbase.regionserver.HRegionFileSystem in project hbase by apache.

the class TestMajorCompactionRequest method makeMockRequest.

private MajorCompactionRequest makeMockRequest(List<StoreFileInfo> storeFiles, boolean references) throws IOException {
    Connection connection = mock(Connection.class);
    RegionInfo regionInfo = mock(RegionInfo.class);
    when(regionInfo.getEncodedName()).thenReturn("HBase");
    when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo"));
    MajorCompactionRequest request = new MajorCompactionRequest(connection, regionInfo, Sets.newHashSet("a"));
    MajorCompactionRequest spy = spy(request);
    HRegionFileSystem fileSystem = mockFileSystem(regionInfo, references, storeFiles);
    doReturn(fileSystem).when(spy).getFileSystem();
    return spy;
}

Example 17 with HRegionFileSystem

use of org.apache.hadoop.hbase.regionserver.HRegionFileSystem in project hbase by apache.

the class RestoreSnapshotHelper method restoreRegion.

/**
 * Restore region by removing files not in the snapshot
 * and adding the missing ones from the snapshot.
 */
private void restoreRegion(final RegionInfo regionInfo, final SnapshotRegionManifest regionManifest, Path regionDir) throws IOException {
    Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles = getRegionHFileReferences(regionManifest);
    String tableName = tableDesc.getTableName().getNameAsString();
    final String snapshotName = snapshotDesc.getName();
    Path regionPath = new Path(tableDir, regionInfo.getEncodedName());
    HRegionFileSystem regionFS = (fs.exists(regionPath)) ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, false) : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, regionInfo);
    // Restore families present in the table
    for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) {
        byte[] family = Bytes.toBytes(familyDir.getName());
        Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
        List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName());
        List<StoreFileInfo> filesToTrack = new ArrayList<>();
        if (snapshotFamilyFiles != null) {
            List<SnapshotRegionManifest.StoreFile> hfilesToAdd = new ArrayList<>();
            for (SnapshotRegionManifest.StoreFile storeFile : snapshotFamilyFiles) {
                if (familyFiles.contains(storeFile.getName())) {
                    // HFile already present
                    familyFiles.remove(storeFile.getName());
                    // no need to restore already present files, but we need to add those to tracker
                    filesToTrack.add(new StoreFileInfo(conf, fs, new Path(familyDir, storeFile.getName()), true));
                } else {
                    // HFile missing
                    hfilesToAdd.add(storeFile);
                }
            }
            // Remove hfiles not present in the snapshot
            for (String hfileName : familyFiles) {
                Path hfile = new Path(familyDir, hfileName);
                if (!fs.getFileStatus(hfile).isDirectory()) {
                    LOG.trace("Removing HFile=" + hfileName + " not present in snapshot=" + snapshotName + " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
                    HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
                }
            }
            // Restore Missing files
            for (SnapshotRegionManifest.StoreFile storeFile : hfilesToAdd) {
                LOG.debug("Restoring missing HFileLink " + storeFile.getName() + " of snapshot=" + snapshotName + " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
                String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
                // mark the reference file to be added to tracker
                filesToTrack.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true));
            }
        } else {
            // Family doesn't exists in the snapshot
            LOG.trace("Removing family=" + Bytes.toString(family) + " in snapshot=" + snapshotName + " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
            HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family);
            fs.delete(familyDir, true);
        }
        StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
        // simply reset list of tracked files with the matching files
        // and the extra one present in the snapshot
        tracker.set(filesToTrack);
    }
    // Add families not present in the table
    for (Map.Entry<String, List<SnapshotRegionManifest.StoreFile>> familyEntry : snapshotFiles.entrySet()) {
        Path familyDir = new Path(regionDir, familyEntry.getKey());
        StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
        List<StoreFileInfo> files = new ArrayList<>();
        if (!fs.mkdirs(familyDir)) {
            throw new IOException("Unable to create familyDir=" + familyDir);
        }
        for (SnapshotRegionManifest.StoreFile storeFile : familyEntry.getValue()) {
            LOG.trace("Adding HFileLink (Not present in the table) " + storeFile.getName() + " of snapshot " + snapshotName + " to table=" + tableName);
            String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
            files.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true));
        }
        tracker.set(files);
    }
}

Example 18 with HRegionFileSystem

use of org.apache.hadoop.hbase.regionserver.HRegionFileSystem in project hbase by apache.

the class SnapshotManifest method addRegion.

/**
   * Creates a 'manifest' for the specified region, by reading directly from the disk.
   * This is used by the "offline snapshot" when the table is disabled.
   */
public void addRegion(final Path tableDir, final HRegionInfo regionInfo) throws IOException {
    // 0. Get the ManifestBuilder/RegionVisitor
    RegionVisitor visitor = createRegionVisitor(desc);
    boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo);
    try {
        Path baseDir = tableDir;
        // Open the RegionFS
        if (isMobRegion) {
            baseDir = FSUtils.getTableDir(MobUtils.getMobHome(conf), regionInfo.getTable());
        }
        HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, baseDir, regionInfo, true);
        monitor.rethrowException();
        // 1. dump region meta info into the snapshot directory
        LOG.debug("Storing region-info for snapshot.");
        Object regionData = visitor.regionOpen(regionInfo);
        monitor.rethrowException();
        // 2. iterate through all the stores in the region
        LOG.debug("Creating references for hfiles");
        // This ensures that we have an atomic view of the directory as long as we have < ls limit
        // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files
        // in batches and may miss files being added/deleted. This could be more robust (iteratively
        // checking to see if we have all the files until we are sure), but the limit is currently
        // 1000 files/batch, far more than the number of store files under a single column family.
        Collection<String> familyNames = regionFs.getFamilies();
        if (familyNames != null) {
            for (String familyName : familyNames) {
                Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName));
                monitor.rethrowException();
                Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName);
                if (storeFiles == null) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("No files under family: " + familyName);
                    }
                    continue;
                }
                // 2.1. build the snapshot reference for the store
                // iterate through all the store's files and create "references".
                addReferenceFiles(visitor, regionData, familyData, storeFiles, false);
                visitor.familyClose(regionData, familyData);
            }
        }
        visitor.regionClose(regionData);
    } catch (IOException e) {
        // the mob directory might not be created yet, so do nothing when it is a mob region
        if (!isMobRegion) {
            throw e;
        }
    }
}

Example 19 with HRegionFileSystem

use of org.apache.hadoop.hbase.regionserver.HRegionFileSystem in project hbase by apache.

the class SnapshotManifestV1 method buildManifestFromDisk.

static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf, final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, true);
    SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
    // 1. dump region meta info into the snapshot directory
    LOG.debug("Storing region-info for snapshot.");
    manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
    // 2. iterate through all the stores in the region
    LOG.debug("Creating references for hfiles");
    // This ensures that we have an atomic view of the directory as long as we have < ls limit
    // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
    // batches and may miss files being added/deleted. This could be more robust (iteratively
    // checking to see if we have all the files until we are sure), but the limit is currently 1000
    // files/batch, far more than the number of store files under a single column family.
    Collection<String> familyNames = regionFs.getFamilies();
    if (familyNames != null) {
        for (String familyName : familyNames) {
            Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName, false);
            if (storeFiles == null) {
                LOG.debug("No files under family: " + familyName);
                continue;
            }
            // 2.1. build the snapshot reference for the store
            SnapshotRegionManifest.FamilyFiles.Builder family = SnapshotRegionManifest.FamilyFiles.newBuilder();
            family.setFamilyName(UnsafeByteOperations.unsafeWrap(Bytes.toBytes(familyName)));
            if (LOG.isDebugEnabled()) {
                LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
            }
            // 2.2. iterate through all the store's files and create "references".
            int i = 0;
            int sz = storeFiles.size();
            for (StoreFileInfo storeFile : storeFiles) {
                // create "reference" to this store file.
                LOG.debug("Adding reference for file (" + (++i) + "/" + sz + "): " + storeFile.getPath());
                SnapshotRegionManifest.StoreFile.Builder sfManifest = SnapshotRegionManifest.StoreFile.newBuilder();
                sfManifest.setName(storeFile.getPath().getName());
                family.addStoreFiles(sfManifest.build());
            }
            manifest.addFamilyFiles(family.build());
        }
    }
    return manifest.build();
}

Example 20 with HRegionFileSystem

use of org.apache.hadoop.hbase.regionserver.HRegionFileSystem in project hbase by apache.

the class SplitTableRegionProcedure method createDaughterRegions.

/**
   * Create daughter regions
   * @param env MasterProcedureEnv
   * @throws IOException
   */
@VisibleForTesting
public void createDaughterRegions(final MasterProcedureEnv env) throws IOException {
    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
    final Path tabledir = FSUtils.getTableDir(mfs.getRootDir(), parentHRI.getTable());
    final FileSystem fs = mfs.getFileSystem();
    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(env.getMasterConfiguration(), fs, tabledir, parentHRI, false);
    regionFs.createSplitsDir();
    Pair<Integer, Integer> expectedReferences = splitStoreFiles(env, regionFs);
    assertReferenceFileCount(fs, expectedReferences.getFirst(), regionFs.getSplitsDir(daughter_1_HRI));
    //Move the files from the temporary .splits to the final /table/region directory
    regionFs.commitDaughterRegion(daughter_1_HRI);
    assertReferenceFileCount(fs, expectedReferences.getFirst(), new Path(tabledir, daughter_1_HRI.getEncodedName()));
    assertReferenceFileCount(fs, expectedReferences.getSecond(), regionFs.getSplitsDir(daughter_2_HRI));
    regionFs.commitDaughterRegion(daughter_2_HRI);
    assertReferenceFileCount(fs, expectedReferences.getSecond(), new Path(tabledir, daughter_2_HRI.getEncodedName()));
}