Examples with SnapshotRegionManifest org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest used on opensource projects

Example 16 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class TableSnapshotInputFormatImpl method getRegionInfosFromManifest.

public static List<RegionInfo> getRegionInfosFromManifest(SnapshotManifest manifest) {
    List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
    if (regionManifests == null) {
        throw new IllegalArgumentException("Snapshot seems empty");
    }
    List<RegionInfo> regionInfos = Lists.newArrayListWithCapacity(regionManifests.size());
    for (SnapshotRegionManifest regionManifest : regionManifests) {
        RegionInfo hri = ProtobufUtil.toRegionInfo(regionManifest.getRegionInfo());
        if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) {
            continue;
        }
        regionInfos.add(hri);
    }
    return regionInfos;
}

Example 17 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class SnapshotManifest method convertToV2SingleManifest.

/*
   * In case of rolling-upgrade, we try to read all the formats and build
   * the snapshot with the latest format.
   */
private void convertToV2SingleManifest() throws IOException {
    // Try to load v1 and v2 regions
    List<SnapshotRegionManifest> v1Regions, v2Regions;
    ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
    setStatusMsg("Loading Region manifests for " + this.desc.getName());
    try {
        v1Regions = SnapshotManifestV1.loadRegionManifests(conf, tpool, workingDirFs, workingDir, desc);
        v2Regions = SnapshotManifestV2.loadRegionManifests(conf, tpool, workingDirFs, workingDir, desc, manifestSizeLimit);
        SnapshotDataManifest.Builder dataManifestBuilder = SnapshotDataManifest.newBuilder();
        dataManifestBuilder.setTableSchema(ProtobufUtil.toTableSchema(htd));
        if (v1Regions != null && v1Regions.size() > 0) {
            dataManifestBuilder.addAllRegionManifests(v1Regions);
        }
        if (v2Regions != null && v2Regions.size() > 0) {
            dataManifestBuilder.addAllRegionManifests(v2Regions);
        }
        // Write the v2 Data Manifest.
        // Once the data-manifest is written, the snapshot can be considered complete.
        // Currently snapshots are written in a "temporary" directory and later
        // moved to the "complated" snapshot directory.
        setStatusMsg("Writing data manifest for " + this.desc.getName());
        SnapshotDataManifest dataManifest = dataManifestBuilder.build();
        writeDataManifest(dataManifest);
        this.regionManifests = dataManifest.getRegionManifestsList();
        // Remove the region manifests. Everything is now in the data-manifest.
        // The delete operation is "relaxed", unless we get an exception we keep going.
        // The extra files in the snapshot directory will not give any problem,
        // since they have the same content as the data manifest, and even by re-reading
        // them we will get the same information.
        int totalDeletes = 0;
        ExecutorCompletionService<Void> completionService = new ExecutorCompletionService<>(tpool);
        if (v1Regions != null) {
            for (SnapshotRegionManifest regionManifest : v1Regions) {
                ++totalDeletes;
                completionService.submit(() -> {
                    SnapshotManifestV1.deleteRegionManifest(workingDirFs, workingDir, regionManifest);
                    return null;
                });
            }
        }
        if (v2Regions != null) {
            for (SnapshotRegionManifest regionManifest : v2Regions) {
                ++totalDeletes;
                completionService.submit(() -> {
                    SnapshotManifestV2.deleteRegionManifest(workingDirFs, workingDir, regionManifest);
                    return null;
                });
            }
        }
        // Wait for the deletes to finish.
        for (int i = 0; i < totalDeletes; i++) {
            try {
                completionService.take().get();
            } catch (InterruptedException ie) {
                throw new InterruptedIOException(ie.getMessage());
            } catch (ExecutionException e) {
                throw new IOException("Error deleting region manifests", e.getCause());
            }
        }
    } finally {
        tpool.shutdown();
    }
}

Example 18 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class SnapshotManifestV2 method loadRegionManifests.

static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf, final Executor executor, final FileSystem fs, final Path snapshotDir, final SnapshotDescription desc, final int manifestSizeLimit) throws IOException {
    FileStatus[] manifestFiles = CommonFSUtils.listStatus(fs, snapshotDir, new PathFilter() {

        @Override
        public boolean accept(Path path) {
            return path.getName().startsWith(SNAPSHOT_MANIFEST_PREFIX);
        }
    });
    if (manifestFiles == null || manifestFiles.length == 0)
        return null;
    final ExecutorCompletionService<SnapshotRegionManifest> completionService = new ExecutorCompletionService<>(executor);
    for (final FileStatus st : manifestFiles) {
        completionService.submit(new Callable<SnapshotRegionManifest>() {

            @Override
            public SnapshotRegionManifest call() throws IOException {
                try (FSDataInputStream stream = fs.open(st.getPath())) {
                    CodedInputStream cin = CodedInputStream.newInstance(stream);
                    cin.setSizeLimit(manifestSizeLimit);
                    return SnapshotRegionManifest.parseFrom(cin);
                }
            }
        });
    }
    ArrayList<SnapshotRegionManifest> regionsManifest = new ArrayList<>(manifestFiles.length);
    try {
        for (int i = 0; i < manifestFiles.length; ++i) {
            regionsManifest.add(completionService.take().get());
        }
    } catch (InterruptedException e) {
        throw new InterruptedIOException(e.getMessage());
    } catch (ExecutionException e) {
        Throwable t = e.getCause();
        if (t instanceof InvalidProtocolBufferException) {
            throw (InvalidProtocolBufferException) t;
        } else {
            throw new IOException("ExecutionException", e.getCause());
        }
    }
    return regionsManifest;
}

Example 19 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class RestoreSnapshotHelper method cloneRegion.

/**
 * Clone region directory content from the snapshot info.
 *
 * Each region is encoded with the table name, so the cloned region will have
 * a different region name.
 *
 * Instead of copying the hfiles a HFileLink is created.
 *
 * @param regionDir {@link Path} cloned dir
 * @param snapshotRegionInfo
 */
private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, final RegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) throws IOException {
    final String tableName = tableDesc.getTableName().getNameAsString();
    final String snapshotName = snapshotDesc.getName();
    for (SnapshotRegionManifest.FamilyFiles familyFiles : manifest.getFamilyFilesList()) {
        Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
        List<StoreFileInfo> clonedFiles = new ArrayList<>();
        for (SnapshotRegionManifest.StoreFile storeFile : familyFiles.getStoreFilesList()) {
            LOG.info("Adding HFileLink " + storeFile.getName() + " from cloned region " + "in snapshot " + snapshotName + " to table=" + tableName);
            if (MobUtils.isMobRegionInfo(newRegionInfo)) {
                String mobFileName = HFileLink.createHFileLinkName(snapshotRegionInfo, storeFile.getName());
                Path mobPath = new Path(familyDir, mobFileName);
                if (fs.exists(mobPath)) {
                    fs.delete(mobPath, true);
                }
                restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
            } else {
                String file = restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
                clonedFiles.add(new StoreFileInfo(conf, fs, new Path(familyDir, file), true));
            }
        }
        // we don't need to track files under mobdir
        if (!MobUtils.isMobRegionInfo(newRegionInfo)) {
            Path regionPath = new Path(tableDir, newRegionInfo.getEncodedName());
            HRegionFileSystem regionFS = (fs.exists(regionPath)) ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, newRegionInfo, false) : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, newRegionInfo);
            Configuration sftConf = StoreUtils.createStoreConfiguration(conf, tableDesc, tableDesc.getColumnFamily(familyFiles.getFamilyName().toByteArray()));
            StoreFileTracker tracker = StoreFileTrackerFactory.create(sftConf, true, StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
            tracker.set(clonedFiles);
        }
    }
}

Example 20 with SnapshotRegionManifest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest in project hbase by apache.

the class SnapshotManifestV1 method buildManifestFromDisk.

static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf, final FileSystem fs, final Path tableDir, final RegionInfo regionInfo) throws IOException {
    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, true);
    SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
    // 1. dump region meta info into the snapshot directory
    LOG.debug("Storing region-info for snapshot.");
    manifest.setRegionInfo(ProtobufUtil.toRegionInfo(regionInfo));
    // 2. iterate through all the stores in the region
    LOG.debug("Creating references for hfiles");
    // This ensures that we have an atomic view of the directory as long as we have < ls limit
    // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
    // batches and may miss files being added/deleted. This could be more robust (iteratively
    // checking to see if we have all the files until we are sure), but the limit is currently 1000
    // files/batch, far more than the number of store files under a single column family.
    Collection<String> familyNames = regionFs.getFamilies();
    if (familyNames != null) {
        for (String familyName : familyNames) {
            Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName, false);
            if (storeFiles == null) {
                LOG.debug("No files under family: " + familyName);
                continue;
            }
            // 2.1. build the snapshot reference for the store
            SnapshotRegionManifest.FamilyFiles.Builder family = SnapshotRegionManifest.FamilyFiles.newBuilder();
            family.setFamilyName(UnsafeByteOperations.unsafeWrap(Bytes.toBytes(familyName)));
            if (LOG.isDebugEnabled()) {
                LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
            }
            // 2.2. iterate through all the store's files and create "references".
            int i = 0;
            int sz = storeFiles.size();
            for (StoreFileInfo storeFile : storeFiles) {
                // create "reference" to this store file.
                LOG.debug("Adding reference for file (" + (++i) + "/" + sz + "): " + storeFile.getPath());
                SnapshotRegionManifest.StoreFile.Builder sfManifest = SnapshotRegionManifest.StoreFile.newBuilder();
                sfManifest.setName(storeFile.getPath().getName());
                family.addStoreFiles(sfManifest.build());
            }
            manifest.addFamilyFiles(family.build());
        }
    }
    return manifest.build();
}