Examples with SnapshotRegionManifest org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest used on opensource projects

Example 6 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class TestSnapshotManifest method createRegionManifest.

private Path createRegionManifest() throws IOException {
    byte[] startKey = Bytes.toBytes("AAAAAA");
    byte[] stopKey = Bytes.toBytes("BBBBBB");
    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(TABLE_NAME).build();
    SnapshotRegionManifest.Builder dataRegionManifestBuilder = SnapshotRegionManifest.newBuilder();
    dataRegionManifestBuilder.setRegionInfo(ProtobufUtil.toRegionInfo(regionInfo));
    for (ColumnFamilyDescriptor hcd : builder.getTableDescriptor().getColumnFamilies()) {
        SnapshotRegionManifest.FamilyFiles.Builder family = SnapshotRegionManifest.FamilyFiles.newBuilder();
        family.setFamilyName(UnsafeByteOperations.unsafeWrap(hcd.getName()));
        for (int j = 0; j < TEST_NUM_REGIONFILES; ++j) {
            SnapshotRegionManifest.StoreFile.Builder sfManifest = SnapshotRegionManifest.StoreFile.newBuilder();
            sfManifest.setName(String.format("%064d", j));
            sfManifest.setFileSize(j * 1024);
            family.addStoreFiles(sfManifest.build());
        }
        dataRegionManifestBuilder.addFamilyFiles(family.build());
    }
    SnapshotRegionManifest manifest = dataRegionManifestBuilder.build();
    Path regionPath = new Path(snapshotDir, SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX + regionInfo.getEncodedName());
    FSDataOutputStream stream = fs.create(regionPath);
    try {
        manifest.writeTo(stream);
    } finally {
        stream.close();
    }
    return regionPath;
}

Example 7 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class SnapshotTestingUtils method confirmSnapshotValid.

/**
 * Confirm that the snapshot contains references to all the files that should
 * be in the snapshot. This method also perform some redundant check like
 * the existence of the snapshotinfo or the regioninfo which are done always
 * by the MasterSnapshotVerifier, at the end of the snapshot operation.
 */
public static void confirmSnapshotValid(SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies, Path rootDir, Admin admin, FileSystem fs) throws IOException {
    final Configuration conf = admin.getConfiguration();
    // check snapshot dir
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDescriptor, rootDir);
    assertTrue("target snapshot directory, '" + snapshotDir + "', doesn't exist.", fs.exists(snapshotDir));
    SnapshotProtos.SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
    // Extract regions and families with store files
    final Set<byte[]> snapshotFamilies = new TreeSet<>(Bytes.BYTES_COMPARATOR);
    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
    Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
    for (SnapshotRegionManifest regionManifest : regionManifests.values()) {
        SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest, new SnapshotReferenceUtil.StoreFileVisitor() {

            @Override
            public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
                snapshotFamilies.add(Bytes.toBytes(family));
            }
        });
    }
    // Verify that there are store files in the specified families
    if (nonEmptyTestFamilies != null) {
        for (final byte[] familyName : nonEmptyTestFamilies) {
            assertTrue("Expected snapshot to contain family '" + Bytes.toString(familyName) + "', but it does not.", snapshotFamilies.contains(familyName));
        }
    }
    // Verify that there are no store files in the specified families
    if (emptyTestFamilies != null) {
        for (final byte[] familyName : emptyTestFamilies) {
            assertFalse("Expected snapshot to skip empty family '" + Bytes.toString(familyName) + "', but it is present.", snapshotFamilies.contains(familyName));
        }
    }
    // check the region snapshot for all the regions
    List<RegionInfo> regions = admin.getRegions(tableName);
    // remove the non-default regions
    RegionReplicaUtil.removeNonDefaultRegions(regions);
    boolean hasMob = regionManifests.containsKey(MobUtils.getMobRegionInfo(tableName).getEncodedName());
    if (hasMob) {
        assertEquals("Wrong number of regions.", regions.size(), regionManifests.size() - 1);
    } else {
        // if create snapshot when table splitting, parent region will be included to the snapshot
        // region manifest. we should exclude the parent regions.
        int regionCountExclusiveSplitParent = 0;
        for (SnapshotRegionManifest snapshotRegionManifest : regionManifests.values()) {
            RegionInfo hri = ProtobufUtil.toRegionInfo(snapshotRegionManifest.getRegionInfo());
            if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) {
                continue;
            }
            regionCountExclusiveSplitParent++;
        }
        assertEquals("Wrong number of regions.", regions.size(), regionCountExclusiveSplitParent);
    }
    // Verify Regions (redundant check, see MasterSnapshotVerifier)
    for (RegionInfo info : regions) {
        String regionName = info.getEncodedName();
        assertTrue("Missing region name: '" + regionName + "'", regionManifests.containsKey(regionName));
    }
}

Example 8 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class SnapshotManifest method getRegionManifestsMap.

/**
 * Get all the Region Manifest from the snapshot.
 * This is an helper to get a map with the region encoded name
 */
public Map<String, SnapshotRegionManifest> getRegionManifestsMap() {
    if (regionManifests == null || regionManifests.isEmpty())
        return null;
    HashMap<String, SnapshotRegionManifest> regionsMap = new HashMap<>(regionManifests.size());
    for (SnapshotRegionManifest manifest : regionManifests) {
        String regionName = getRegionNameFromManifest(manifest);
        regionsMap.put(regionName, manifest);
    }
    return regionsMap;
}

Example 9 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class SnapshotReferenceUtil method concurrentVisitReferencedFiles.

public static void concurrentVisitReferencedFiles(final Configuration conf, final FileSystem fs, final SnapshotManifest manifest, final String desc, final StoreFileVisitor visitor) throws IOException {
    final Path snapshotDir = manifest.getSnapshotDir();
    List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
    if (regionManifests == null || regionManifests.isEmpty()) {
        LOG.debug("No manifest files present: " + snapshotDir);
        return;
    }
    ExecutorService exec = SnapshotManifest.createExecutor(conf, desc);
    try {
        concurrentVisitReferencedFiles(conf, fs, manifest, exec, visitor);
    } finally {
        exec.shutdown();
    }
}

Example 10 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class RestoreSnapshotHelper method restoreRegion.

/**
 * Restore region by removing files not in the snapshot
 * and adding the missing ones from the snapshot.
 */
private void restoreRegion(final RegionInfo regionInfo, final SnapshotRegionManifest regionManifest, Path regionDir) throws IOException {
    Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles = getRegionHFileReferences(regionManifest);
    String tableName = tableDesc.getTableName().getNameAsString();
    final String snapshotName = snapshotDesc.getName();
    Path regionPath = new Path(tableDir, regionInfo.getEncodedName());
    HRegionFileSystem regionFS = (fs.exists(regionPath)) ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, false) : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, regionInfo);
    // Restore families present in the table
    for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) {
        byte[] family = Bytes.toBytes(familyDir.getName());
        Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
        List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName());
        List<StoreFileInfo> filesToTrack = new ArrayList<>();
        if (snapshotFamilyFiles != null) {
            List<SnapshotRegionManifest.StoreFile> hfilesToAdd = new ArrayList<>();
            for (SnapshotRegionManifest.StoreFile storeFile : snapshotFamilyFiles) {
                if (familyFiles.contains(storeFile.getName())) {
                    // HFile already present
                    familyFiles.remove(storeFile.getName());
                    // no need to restore already present files, but we need to add those to tracker
                    filesToTrack.add(new StoreFileInfo(conf, fs, new Path(familyDir, storeFile.getName()), true));
                } else {
                    // HFile missing
                    hfilesToAdd.add(storeFile);
                }
            }
            // Remove hfiles not present in the snapshot
            for (String hfileName : familyFiles) {
                Path hfile = new Path(familyDir, hfileName);
                if (!fs.getFileStatus(hfile).isDirectory()) {
                    LOG.trace("Removing HFile=" + hfileName + " not present in snapshot=" + snapshotName + " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
                    HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
                }
            }
            // Restore Missing files
            for (SnapshotRegionManifest.StoreFile storeFile : hfilesToAdd) {
                LOG.debug("Restoring missing HFileLink " + storeFile.getName() + " of snapshot=" + snapshotName + " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
                String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
                // mark the reference file to be added to tracker
                filesToTrack.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true));
            }
        } else {
            // Family doesn't exists in the snapshot
            LOG.trace("Removing family=" + Bytes.toString(family) + " in snapshot=" + snapshotName + " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
            HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family);
            fs.delete(familyDir, true);
        }
        StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
        // simply reset list of tracked files with the matching files
        // and the extra one present in the snapshot
        tracker.set(filesToTrack);
    }
    // Add families not present in the table
    for (Map.Entry<String, List<SnapshotRegionManifest.StoreFile>> familyEntry : snapshotFiles.entrySet()) {
        Path familyDir = new Path(regionDir, familyEntry.getKey());
        StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
        List<StoreFileInfo> files = new ArrayList<>();
        if (!fs.mkdirs(familyDir)) {
            throw new IOException("Unable to create familyDir=" + familyDir);
        }
        for (SnapshotRegionManifest.StoreFile storeFile : familyEntry.getValue()) {
            LOG.trace("Adding HFileLink (Not present in the table) " + storeFile.getName() + " of snapshot " + snapshotName + " to table=" + tableName);
            String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
            files.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true));
        }
        tracker.set(files);
    }
}