Examples with SnapshotRegionManifest org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest used on opensource projects

Example 11 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class RestoreSnapshotHelper method restoreHdfsRegions.

private RestoreMetaChanges restoreHdfsRegions(final ThreadPoolExecutor exec) throws IOException {
    LOG.info("starting restore table regions using snapshot=" + snapshotDesc);
    Map<String, SnapshotRegionManifest> regionManifests = snapshotManifest.getRegionManifestsMap();
    if (regionManifests == null) {
        LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
        return null;
    }
    RestoreMetaChanges metaChanges = new RestoreMetaChanges(tableDesc, parentsMap);
    // Take a copy of the manifest.keySet() since we are going to modify
    // this instance, by removing the regions already present in the restore dir.
    Set<String> regionNames = new HashSet<>(regionManifests.keySet());
    List<RegionInfo> tableRegions = getTableRegions();
    RegionInfo mobRegion = MobUtils.getMobRegionInfo(snapshotManifest.getTableDescriptor().getTableName());
    if (tableRegions != null) {
        // restore the mob region in case
        if (regionNames.contains(mobRegion.getEncodedName())) {
            monitor.rethrowException();
            status.setStatus("Restoring mob region...");
            List<RegionInfo> mobRegions = new ArrayList<>(1);
            mobRegions.add(mobRegion);
            restoreHdfsMobRegions(exec, regionManifests, mobRegions);
            regionNames.remove(mobRegion.getEncodedName());
            status.setStatus("Finished restoring mob region.");
        }
    }
    if (regionNames.contains(mobRegion.getEncodedName())) {
        // add the mob region
        monitor.rethrowException();
        status.setStatus("Cloning mob region...");
        cloneHdfsMobRegion(regionManifests, mobRegion);
        regionNames.remove(mobRegion.getEncodedName());
        status.setStatus("Finished cloning mob region.");
    }
    // NOTE: we rely upon the region name as: "table name, start key, end key"
    if (tableRegions != null) {
        monitor.rethrowException();
        for (RegionInfo regionInfo : tableRegions) {
            String regionName = regionInfo.getEncodedName();
            if (regionNames.contains(regionName)) {
                LOG.info("region to restore: " + regionName);
                regionNames.remove(regionName);
                metaChanges.addRegionToRestore(ProtobufUtil.toRegionInfo(regionManifests.get(regionName).getRegionInfo()));
            } else {
                LOG.info("region to remove: " + regionName);
                metaChanges.addRegionToRemove(regionInfo);
            }
        }
    }
    // Regions to Add: present in the snapshot but not in the current table
    List<RegionInfo> regionsToAdd = new ArrayList<>(regionNames.size());
    if (regionNames.size() > 0) {
        monitor.rethrowException();
        for (String regionName : regionNames) {
            LOG.info("region to add: " + regionName);
            regionsToAdd.add(ProtobufUtil.toRegionInfo(regionManifests.get(regionName).getRegionInfo()));
        }
    }
    // Create new regions cloning from the snapshot
    // HBASE-19980: We need to call cloneHdfsRegions() before restoreHdfsRegions() because
    // regionsMap is constructed in cloneHdfsRegions() and it can be used in restoreHdfsRegions().
    monitor.rethrowException();
    status.setStatus("Cloning regions...");
    RegionInfo[] clonedRegions = cloneHdfsRegions(exec, regionManifests, regionsToAdd);
    metaChanges.setNewRegions(clonedRegions);
    status.setStatus("Finished cloning regions.");
    // Restore regions using the snapshot data
    monitor.rethrowException();
    status.setStatus("Restoring table regions...");
    restoreHdfsRegions(exec, regionManifests, metaChanges.getRegionsToRestore());
    status.setStatus("Finished restoring all table regions.");
    // Remove regions from the current table
    monitor.rethrowException();
    status.setStatus("Starting to delete excess regions from table");
    removeHdfsRegions(exec, metaChanges.getRegionsToRemove());
    status.setStatus("Finished deleting excess regions from table.");
    LOG.info("finishing restore table regions using snapshot=" + snapshotDesc);
    return metaChanges;
}

Example 12 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class SnapshotManifestV1 method loadRegionManifests.

static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf, final Executor executor, final FileSystem fs, final Path snapshotDir, final SnapshotDescription desc) throws IOException {
    FileStatus[] regions = CommonFSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
    if (regions == null) {
        LOG.debug("No regions under directory:" + snapshotDir);
        return null;
    }
    final ExecutorCompletionService<SnapshotRegionManifest> completionService = new ExecutorCompletionService<>(executor);
    for (final FileStatus region : regions) {
        completionService.submit(new Callable<SnapshotRegionManifest>() {

            @Override
            public SnapshotRegionManifest call() throws IOException {
                RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, region.getPath());
                return buildManifestFromDisk(conf, fs, snapshotDir, hri);
            }
        });
    }
    ArrayList<SnapshotRegionManifest> regionsManifest = new ArrayList<>(regions.length);
    try {
        for (int i = 0; i < regions.length; ++i) {
            regionsManifest.add(completionService.take().get());
        }
    } catch (InterruptedException e) {
        throw new InterruptedIOException(e.getMessage());
    } catch (ExecutionException e) {
        throw new IOException(e.getCause());
    }
    return regionsManifest;
}

Example 13 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class SnapshotManifestV1 method buildManifestFromDisk.

static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf, final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, true);
    SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
    // 1. dump region meta info into the snapshot directory
    LOG.debug("Storing region-info for snapshot.");
    manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
    // 2. iterate through all the stores in the region
    LOG.debug("Creating references for hfiles");
    // This ensures that we have an atomic view of the directory as long as we have < ls limit
    // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
    // batches and may miss files being added/deleted. This could be more robust (iteratively
    // checking to see if we have all the files until we are sure), but the limit is currently 1000
    // files/batch, far more than the number of store files under a single column family.
    Collection<String> familyNames = regionFs.getFamilies();
    if (familyNames != null) {
        for (String familyName : familyNames) {
            Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName, false);
            if (storeFiles == null) {
                LOG.debug("No files under family: " + familyName);
                continue;
            }
            // 2.1. build the snapshot reference for the store
            SnapshotRegionManifest.FamilyFiles.Builder family = SnapshotRegionManifest.FamilyFiles.newBuilder();
            family.setFamilyName(UnsafeByteOperations.unsafeWrap(Bytes.toBytes(familyName)));
            if (LOG.isDebugEnabled()) {
                LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
            }
            // 2.2. iterate through all the store's files and create "references".
            int i = 0;
            int sz = storeFiles.size();
            for (StoreFileInfo storeFile : storeFiles) {
                // create "reference" to this store file.
                LOG.debug("Adding reference for file (" + (++i) + "/" + sz + "): " + storeFile.getPath());
                SnapshotRegionManifest.StoreFile.Builder sfManifest = SnapshotRegionManifest.StoreFile.newBuilder();
                sfManifest.setName(storeFile.getPath().getName());
                family.addStoreFiles(sfManifest.build());
            }
            manifest.addFamilyFiles(family.build());
        }
    }
    return manifest.build();
}

Example 14 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class RestoreSnapshotHelper method cloneRegion.

/**
   * Clone region directory content from the snapshot info.
   *
   * Each region is encoded with the table name, so the cloned region will have
   * a different region name.
   *
   * Instead of copying the hfiles a HFileLink is created.
   *
   * @param regionDir {@link Path} cloned dir
   * @param snapshotRegionInfo
   */
private void cloneRegion(final Path regionDir, final HRegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) throws IOException {
    final String tableName = tableDesc.getTableName().getNameAsString();
    for (SnapshotRegionManifest.FamilyFiles familyFiles : manifest.getFamilyFilesList()) {
        Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
        for (SnapshotRegionManifest.StoreFile storeFile : familyFiles.getStoreFilesList()) {
            LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
            restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
        }
    }
}

Example 15 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class MasterSnapshotVerifier method verifyRegions.

/**
 * Check that all the regions in the snapshot are valid, and accounted for.
 * @param manifest snapshot manifest to inspect
 * @throws IOException if we can't reach hbase:meta or read the files from the FS
 */
private void verifyRegions(final SnapshotManifest manifest) throws IOException {
    List<RegionInfo> regions = services.getAssignmentManager().getTableRegions(tableName, false);
    // Remove the non-default regions
    RegionReplicaUtil.removeNonDefaultRegions(regions);
    Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
    if (regionManifests == null) {
        String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty";
        LOG.error(msg);
        throw new CorruptedSnapshotException(msg);
    }
    String errorMsg = "";
    boolean hasMobStore = false;
    // the mob region has a special name, it could be found by the region name.
    if (regionManifests.get(MobUtils.getMobRegionInfo(tableName).getEncodedName()) != null) {
        hasMobStore = true;
    }
    int realRegionCount = hasMobStore ? regionManifests.size() - 1 : regionManifests.size();
    if (realRegionCount != regions.size()) {
        errorMsg = "Regions moved during the snapshot '" + ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" + regions.size() + " snapshotted=" + realRegionCount + ".";
        LOG.error(errorMsg);
    }
    // Verify RegionInfo
    for (RegionInfo region : regions) {
        SnapshotRegionManifest regionManifest = regionManifests.get(region.getEncodedName());
        if (regionManifest == null) {
            // could happen due to a move or split race.
            String mesg = " No snapshot region directory found for region:" + region;
            if (errorMsg.isEmpty())
                errorMsg = mesg;
            LOG.error(mesg);
            continue;
        }
        verifyRegionInfo(region, regionManifest);
    }
    if (!errorMsg.isEmpty()) {
        throw new CorruptedSnapshotException(errorMsg);
    }
    // Verify Snapshot HFiles
    // Requires the root directory file system as HFiles are stored in the root directory
    SnapshotReferenceUtil.verifySnapshot(services.getConfiguration(), CommonFSUtils.getRootDirFileSystem(services.getConfiguration()), manifest);
}