Examples with StoreFileInfo org.apache.hadoop.hbase.regionserver.StoreFileInfo used on opensource projects

Example 16 with StoreFileInfo

use of org.apache.hadoop.hbase.regionserver.StoreFileInfo in project hbase by apache.

the class SnapshotManifest method addReferenceFiles.

private void addReferenceFiles(RegionVisitor visitor, Object regionData, Object familyData, Collection<StoreFileInfo> storeFiles, boolean isMob) throws IOException {
    final String fileType = isMob ? "mob file" : "hfile";
    if (LOG.isDebugEnabled()) {
        LOG.debug(String.format("Adding snapshot references for %s %ss", storeFiles, fileType));
    }
    int i = 0;
    int sz = storeFiles.size();
    for (StoreFileInfo storeFile : storeFiles) {
        monitor.rethrowException();
        LOG.debug(String.format("Adding reference for %s (%d/%d): %s", fileType, ++i, sz, storeFile.getPath()));
        // create "reference" to this store file.
        visitor.storeFile(regionData, familyData, storeFile);
    }
}

Example 17 with StoreFileInfo

use of org.apache.hadoop.hbase.regionserver.StoreFileInfo in project hbase by apache.

the class SnapshotManifest method getStoreFiles.

private List<StoreFileInfo> getStoreFiles(Path storeDir) throws IOException {
    FileStatus[] stats = CommonFSUtils.listStatus(rootFs, storeDir);
    if (stats == null)
        return null;
    ArrayList<StoreFileInfo> storeFiles = new ArrayList<>(stats.length);
    for (int i = 0; i < stats.length; ++i) {
        storeFiles.add(new StoreFileInfo(conf, rootFs, stats[i]));
    }
    return storeFiles;
}

Example 18 with StoreFileInfo

use of org.apache.hadoop.hbase.regionserver.StoreFileInfo in project hbase by apache.

the class RestoreSnapshotHelper method cloneRegion.

/**
 * Clone region directory content from the snapshot info.
 *
 * Each region is encoded with the table name, so the cloned region will have
 * a different region name.
 *
 * Instead of copying the hfiles a HFileLink is created.
 *
 * @param regionDir {@link Path} cloned dir
 * @param snapshotRegionInfo
 */
private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, final RegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) throws IOException {
    final String tableName = tableDesc.getTableName().getNameAsString();
    final String snapshotName = snapshotDesc.getName();
    for (SnapshotRegionManifest.FamilyFiles familyFiles : manifest.getFamilyFilesList()) {
        Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
        List<StoreFileInfo> clonedFiles = new ArrayList<>();
        for (SnapshotRegionManifest.StoreFile storeFile : familyFiles.getStoreFilesList()) {
            LOG.info("Adding HFileLink " + storeFile.getName() + " from cloned region " + "in snapshot " + snapshotName + " to table=" + tableName);
            if (MobUtils.isMobRegionInfo(newRegionInfo)) {
                String mobFileName = HFileLink.createHFileLinkName(snapshotRegionInfo, storeFile.getName());
                Path mobPath = new Path(familyDir, mobFileName);
                if (fs.exists(mobPath)) {
                    fs.delete(mobPath, true);
                }
                restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
            } else {
                String file = restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
                clonedFiles.add(new StoreFileInfo(conf, fs, new Path(familyDir, file), true));
            }
        }
        // we don't need to track files under mobdir
        if (!MobUtils.isMobRegionInfo(newRegionInfo)) {
            Path regionPath = new Path(tableDir, newRegionInfo.getEncodedName());
            HRegionFileSystem regionFS = (fs.exists(regionPath)) ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, newRegionInfo, false) : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, newRegionInfo);
            Configuration sftConf = StoreUtils.createStoreConfiguration(conf, tableDesc, tableDesc.getColumnFamily(familyFiles.getFamilyName().toByteArray()));
            StoreFileTracker tracker = StoreFileTrackerFactory.create(sftConf, true, StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
            tracker.set(clonedFiles);
        }
    }
}

Example 19 with StoreFileInfo

use of org.apache.hadoop.hbase.regionserver.StoreFileInfo in project hbase by apache.

the class SnapshotManifestV1 method buildManifestFromDisk.

static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf, final FileSystem fs, final Path tableDir, final RegionInfo regionInfo) throws IOException {
    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, true);
    SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
    // 1. dump region meta info into the snapshot directory
    LOG.debug("Storing region-info for snapshot.");
    manifest.setRegionInfo(ProtobufUtil.toRegionInfo(regionInfo));
    // 2. iterate through all the stores in the region
    LOG.debug("Creating references for hfiles");
    // This ensures that we have an atomic view of the directory as long as we have < ls limit
    // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
    // batches and may miss files being added/deleted. This could be more robust (iteratively
    // checking to see if we have all the files until we are sure), but the limit is currently 1000
    // files/batch, far more than the number of store files under a single column family.
    Collection<String> familyNames = regionFs.getFamilies();
    if (familyNames != null) {
        for (String familyName : familyNames) {
            Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName, false);
            if (storeFiles == null) {
                LOG.debug("No files under family: " + familyName);
                continue;
            }
            // 2.1. build the snapshot reference for the store
            SnapshotRegionManifest.FamilyFiles.Builder family = SnapshotRegionManifest.FamilyFiles.newBuilder();
            family.setFamilyName(UnsafeByteOperations.unsafeWrap(Bytes.toBytes(familyName)));
            if (LOG.isDebugEnabled()) {
                LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
            }
            // 2.2. iterate through all the store's files and create "references".
            int i = 0;
            int sz = storeFiles.size();
            for (StoreFileInfo storeFile : storeFiles) {
                // create "reference" to this store file.
                LOG.debug("Adding reference for file (" + (++i) + "/" + sz + "): " + storeFile.getPath());
                SnapshotRegionManifest.StoreFile.Builder sfManifest = SnapshotRegionManifest.StoreFile.newBuilder();
                sfManifest.setName(storeFile.getPath().getName());
                family.addStoreFiles(sfManifest.build());
            }
            manifest.addFamilyFiles(family.build());
        }
    }
    return manifest.build();
}

Example 20 with StoreFileInfo

use of org.apache.hadoop.hbase.regionserver.StoreFileInfo in project hbase by apache.

the class TestSnapshotStoreFileSize method testIsStoreFileSizeMatchFilesystemAndManifest.

@Test
public void testIsStoreFileSizeMatchFilesystemAndManifest() throws IOException {
    admin = UTIL.getAdmin();
    fs = UTIL.getTestFileSystem();
    UTIL.createTable(TABLE_NAME, FAMILY_NAME.getBytes());
    Table table = admin.getConnection().getTable(TABLE_NAME);
    UTIL.loadRandomRows(table, FAMILY_NAME.getBytes(), 3, 1000);
    admin.snapshot(SNAPSHOT_NAME, TABLE_NAME);
    Map<String, Long> storeFileInfoFromManifest = new HashMap<String, Long>();
    Map<String, Long> storeFileInfoFromFS = new HashMap<String, Long>();
    String storeFileName = "";
    long storeFilesize = 0L;
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(SNAPSHOT_NAME, UTIL.getDefaultRootDirPath());
    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
    SnapshotManifest snaphotManifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
    List<SnapshotRegionManifest> regionManifest = snaphotManifest.getRegionManifests();
    for (int i = 0; i < regionManifest.size(); i++) {
        SnapshotRegionManifest.FamilyFiles family = regionManifest.get(i).getFamilyFiles(0);
        List<SnapshotRegionManifest.StoreFile> storeFiles = family.getStoreFilesList();
        for (int j = 0; j < storeFiles.size(); j++) {
            storeFileName = storeFiles.get(j).getName();
            storeFilesize = storeFiles.get(j).getFileSize();
            storeFileInfoFromManifest.put(storeFileName, storeFilesize);
        }
    }
    List<RegionInfo> regionsInfo = admin.getRegions(TABLE_NAME);
    Path path = CommonFSUtils.getTableDir(UTIL.getDefaultRootDirPath(), TABLE_NAME);
    for (RegionInfo regionInfo : regionsInfo) {
        HRegionFileSystem hRegionFileSystem = HRegionFileSystem.openRegionFromFileSystem(conf, fs, path, regionInfo, true);
        Collection<StoreFileInfo> storeFilesFS = hRegionFileSystem.getStoreFiles(FAMILY_NAME);
        Iterator<StoreFileInfo> sfIterator = storeFilesFS.iterator();
        while (sfIterator.hasNext()) {
            StoreFileInfo sfi = sfIterator.next();
            FileStatus[] fileStatus = CommonFSUtils.listStatus(fs, sfi.getPath());
            storeFileName = fileStatus[0].getPath().getName();
            storeFilesize = fileStatus[0].getLen();
            storeFileInfoFromFS.put(storeFileName, storeFilesize);
        }
    }
    Assert.assertEquals(storeFileInfoFromManifest, storeFileInfoFromFS);
}