Examples with SnapshotRegionManifest org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest used on opensource projects

Example 21 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class SnapshotReferenceUtil method concurrentVisitReferencedFiles.

public static void concurrentVisitReferencedFiles(final Configuration conf, final FileSystem fs, final SnapshotManifest manifest, final ExecutorService exec, final StoreFileVisitor visitor) throws IOException {
    final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription();
    final Path snapshotDir = manifest.getSnapshotDir();
    List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
    if (regionManifests == null || regionManifests.isEmpty()) {
        LOG.debug("No manifest files present: " + snapshotDir);
        return;
    }
    final ExecutorCompletionService<Void> completionService = new ExecutorCompletionService<>(exec);
    for (final SnapshotRegionManifest regionManifest : regionManifests) {
        completionService.submit(new Callable<Void>() {

            @Override
            public Void call() throws IOException {
                visitRegionStoreFiles(regionManifest, visitor);
                return null;
            }
        });
    }
    try {
        for (int i = 0; i < regionManifests.size(); ++i) {
            completionService.take().get();
        }
    } catch (InterruptedException e) {
        throw new InterruptedIOException(e.getMessage());
    } catch (ExecutionException e) {
        if (e.getCause() instanceof CorruptedSnapshotException) {
            throw new CorruptedSnapshotException(e.getCause().getMessage(), ProtobufUtil.createSnapshotDesc(snapshotDesc));
        } else {
            throw new IOException(e.getCause());
        }
    }
}

Example 22 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class SnapshotReferenceUtil method visitTableStoreFiles.

/**
 *©
 * Iterate over the snapshot store files
 *
 * @param conf The current {@link Configuration} instance.
 * @param fs {@link FileSystem}
 * @param snapshotDir {@link Path} to the Snapshot directory
 * @param desc the {@link SnapshotDescription} of the snapshot to verify
 * @param visitor callback object to get the store files
 * @throws IOException if an error occurred while scanning the directory
 */
static void visitTableStoreFiles(final Configuration conf, final FileSystem fs, final Path snapshotDir, final SnapshotDescription desc, final StoreFileVisitor visitor) throws IOException {
    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
    List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
    if (regionManifests == null || regionManifests.isEmpty()) {
        LOG.debug("No manifest files present: " + snapshotDir);
        return;
    }
    for (SnapshotRegionManifest regionManifest : regionManifests) {
        visitRegionStoreFiles(regionManifest, visitor);
    }
}

Example 23 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class TestSnapshotStoreFileSize method testIsStoreFileSizeMatchFilesystemAndManifest.

@Test
public void testIsStoreFileSizeMatchFilesystemAndManifest() throws IOException {
    admin = UTIL.getAdmin();
    fs = UTIL.getTestFileSystem();
    UTIL.createTable(TABLE_NAME, FAMILY_NAME.getBytes());
    Table table = admin.getConnection().getTable(TABLE_NAME);
    UTIL.loadRandomRows(table, FAMILY_NAME.getBytes(), 3, 1000);
    admin.snapshot(SNAPSHOT_NAME, TABLE_NAME);
    Map<String, Long> storeFileInfoFromManifest = new HashMap<String, Long>();
    Map<String, Long> storeFileInfoFromFS = new HashMap<String, Long>();
    String storeFileName = "";
    long storeFilesize = 0L;
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(SNAPSHOT_NAME, UTIL.getDefaultRootDirPath());
    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
    SnapshotManifest snaphotManifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
    List<SnapshotRegionManifest> regionManifest = snaphotManifest.getRegionManifests();
    for (int i = 0; i < regionManifest.size(); i++) {
        SnapshotRegionManifest.FamilyFiles family = regionManifest.get(i).getFamilyFiles(0);
        List<SnapshotRegionManifest.StoreFile> storeFiles = family.getStoreFilesList();
        for (int j = 0; j < storeFiles.size(); j++) {
            storeFileName = storeFiles.get(j).getName();
            storeFilesize = storeFiles.get(j).getFileSize();
            storeFileInfoFromManifest.put(storeFileName, storeFilesize);
        }
    }
    List<RegionInfo> regionsInfo = admin.getRegions(TABLE_NAME);
    Path path = CommonFSUtils.getTableDir(UTIL.getDefaultRootDirPath(), TABLE_NAME);
    for (RegionInfo regionInfo : regionsInfo) {
        HRegionFileSystem hRegionFileSystem = HRegionFileSystem.openRegionFromFileSystem(conf, fs, path, regionInfo, true);
        Collection<StoreFileInfo> storeFilesFS = hRegionFileSystem.getStoreFiles(FAMILY_NAME);
        Iterator<StoreFileInfo> sfIterator = storeFilesFS.iterator();
        while (sfIterator.hasNext()) {
            StoreFileInfo sfi = sfIterator.next();
            FileStatus[] fileStatus = CommonFSUtils.listStatus(fs, sfi.getPath());
            storeFileName = fileStatus[0].getPath().getName();
            storeFilesize = fileStatus[0].getLen();
            storeFileInfoFromFS.put(storeFileName, storeFilesize);
        }
    }
    Assert.assertEquals(storeFileInfoFromManifest, storeFileInfoFromFS);
}