Examples with HRegionFileSystem org.apache.hadoop.hbase.regionserver.HRegionFileSystem used on opensource projects

Example 11 with HRegionFileSystem

use of org.apache.hadoop.hbase.regionserver.HRegionFileSystem in project hbase by apache.

the class MasterRegion method tryMigrate.

private static void tryMigrate(Configuration conf, FileSystem fs, Path tableDir, RegionInfo regionInfo, TableDescriptor oldTd, TableDescriptor newTd) throws IOException {
    Class<? extends StoreFileTracker> oldSft = StoreFileTrackerFactory.getTrackerClass(oldTd.getValue(StoreFileTrackerFactory.TRACKER_IMPL));
    Class<? extends StoreFileTracker> newSft = StoreFileTrackerFactory.getTrackerClass(newTd.getValue(StoreFileTrackerFactory.TRACKER_IMPL));
    if (oldSft.equals(newSft)) {
        LOG.debug("old store file tracker {} is the same with new store file tracker, skip migration", StoreFileTrackerFactory.getStoreFileTrackerName(oldSft));
        if (!oldTd.equals(newTd)) {
            // we may change other things such as adding a new family, so here we still need to persist
            // the new table descriptor
            LOG.info("Update table descriptor from {} to {}", oldTd, newTd);
            FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, newTd, true);
        }
        return;
    }
    LOG.info("Migrate store file tracker from {} to {}", oldSft.getSimpleName(), newSft.getSimpleName());
    HRegionFileSystem hfs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, false);
    for (ColumnFamilyDescriptor oldCfd : oldTd.getColumnFamilies()) {
        StoreFileTracker oldTracker = StoreFileTrackerFactory.create(conf, oldTd, oldCfd, hfs);
        StoreFileTracker newTracker = StoreFileTrackerFactory.create(conf, oldTd, oldCfd, hfs);
        List<StoreFileInfo> files = oldTracker.load();
        LOG.debug("Store file list for {}: {}", oldCfd.getNameAsString(), files);
        newTracker.set(oldTracker.load());
    }
    // persist the new table descriptor after migration
    LOG.info("Update table descriptor from {} to {}", oldTd, newTd);
    FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, newTd, true);
}

Example 12 with HRegionFileSystem

use of org.apache.hadoop.hbase.regionserver.HRegionFileSystem in project hbase by apache.

the class TestStoreFileListFile method setUp.

@Before
public void setUp() throws IOException {
    testDir = UTIL.getDataTestDir(name.getMethodName());
    HRegionFileSystem hfs = mock(HRegionFileSystem.class);
    when(hfs.getFileSystem()).thenReturn(FileSystem.get(UTIL.getConfiguration()));
    StoreContext ctx = StoreContext.getBuilder().withFamilyStoreDirectoryPath(testDir).withRegionFileSystem(hfs).build();
    storeFileListFile = new StoreFileListFile(ctx);
}

Example 13 with HRegionFileSystem

use of org.apache.hadoop.hbase.regionserver.HRegionFileSystem in project hbase by apache.

the class RegionSplitter method splitScan.

static LinkedList<Pair<byte[], byte[]>> splitScan(LinkedList<Pair<byte[], byte[]>> regionList, final Connection connection, final TableName tableName, SplitAlgorithm splitAlgo) throws IOException, InterruptedException {
    LinkedList<Pair<byte[], byte[]>> finished = Lists.newLinkedList();
    LinkedList<Pair<byte[], byte[]>> logicalSplitting = Lists.newLinkedList();
    LinkedList<Pair<byte[], byte[]>> physicalSplitting = Lists.newLinkedList();
    // Get table info
    Pair<Path, Path> tableDirAndSplitFile = getTableDirAndSplitFile(connection.getConfiguration(), tableName);
    Path tableDir = tableDirAndSplitFile.getFirst();
    FileSystem fs = tableDir.getFileSystem(connection.getConfiguration());
    // Clear the cache to forcibly refresh region information
    connection.clearRegionLocationCache();
    TableDescriptor htd = null;
    try (Table table = connection.getTable(tableName)) {
        htd = table.getDescriptor();
    }
    try (RegionLocator regionLocator = connection.getRegionLocator(tableName)) {
        // for every region that hasn't been verified as a finished split
        for (Pair<byte[], byte[]> region : regionList) {
            byte[] start = region.getFirst();
            byte[] split = region.getSecond();
            // see if the new split daughter region has come online
            try {
                RegionInfo dri = regionLocator.getRegionLocation(split, true).getRegion();
                if (dri.isOffline() || !Bytes.equals(dri.getStartKey(), split)) {
                    logicalSplitting.add(region);
                    continue;
                }
            } catch (NoServerForRegionException nsfre) {
                // NSFRE will occur if the old hbase:meta entry has no server assigned
                LOG.info(nsfre.toString(), nsfre);
                logicalSplitting.add(region);
                continue;
            }
            try {
                // when a daughter region is opened, a compaction is triggered
                // wait until compaction completes for both daughter regions
                LinkedList<RegionInfo> check = Lists.newLinkedList();
                check.add(regionLocator.getRegionLocation(start).getRegion());
                check.add(regionLocator.getRegionLocation(split).getRegion());
                for (RegionInfo hri : check.toArray(new RegionInfo[check.size()])) {
                    byte[] sk = hri.getStartKey();
                    if (sk.length == 0)
                        sk = splitAlgo.firstRow();
                    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(connection.getConfiguration(), fs, tableDir, hri, true);
                    // Check every Column Family for that region -- check does not have references.
                    boolean refFound = false;
                    for (ColumnFamilyDescriptor c : htd.getColumnFamilies()) {
                        if ((refFound = regionFs.hasReferences(c.getNameAsString()))) {
                            break;
                        }
                    }
                    // compaction is completed when all reference files are gone
                    if (!refFound) {
                        check.remove(hri);
                    }
                }
                if (check.isEmpty()) {
                    finished.add(region);
                } else {
                    physicalSplitting.add(region);
                }
            } catch (NoServerForRegionException nsfre) {
                LOG.debug("No Server Exception thrown for: " + splitAlgo.rowToStr(start));
                physicalSplitting.add(region);
                connection.clearRegionLocationCache();
            }
        }
        LOG.debug("Split Scan: " + finished.size() + " finished / " + logicalSplitting.size() + " split wait / " + physicalSplitting.size() + " reference wait");
        return finished;
    }
}

Example 14 with HRegionFileSystem

use of org.apache.hadoop.hbase.regionserver.HRegionFileSystem in project hbase by apache.

the class TestTableSnapshotScanner method testMergeRegion.

@Test
public void testMergeRegion() throws Exception {
    setupCluster();
    TableName tableName = TableName.valueOf("testMergeRegion");
    String snapshotName = tableName.getNameAsString() + "_snapshot";
    Configuration conf = UTIL.getConfiguration();
    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
    // 20s
    long timeout = 20000;
    try (Admin admin = UTIL.getAdmin()) {
        List<String> serverList = admin.getRegionServers().stream().map(sn -> sn.getServerName()).collect(Collectors.toList());
        // create table with 3 regions
        Table table = UTIL.createTable(tableName, FAMILIES, 1, bbb, yyy, 3);
        List<RegionInfo> regions = admin.getRegions(tableName);
        Assert.assertEquals(3, regions.size());
        RegionInfo region0 = regions.get(0);
        RegionInfo region1 = regions.get(1);
        RegionInfo region2 = regions.get(2);
        // put some data in the table
        UTIL.loadTable(table, FAMILIES);
        admin.flush(tableName);
        // wait flush is finished
        UTIL.waitFor(timeout, () -> {
            try {
                Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
                for (RegionInfo region : regions) {
                    Path regionDir = new Path(tableDir, region.getEncodedName());
                    for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) {
                        if (fs.listStatus(familyDir).length != 1) {
                            return false;
                        }
                    }
                }
                return true;
            } catch (IOException e) {
                LOG.warn("Failed check if flush is finished", e);
                return false;
            }
        });
        // merge 2 regions
        admin.compactionSwitch(false, serverList);
        admin.mergeRegionsAsync(region0.getEncodedNameAsBytes(), region1.getEncodedNameAsBytes(), true);
        UTIL.waitFor(timeout, () -> admin.getRegions(tableName).size() == 2);
        List<RegionInfo> mergedRegions = admin.getRegions(tableName);
        RegionInfo mergedRegion = mergedRegions.get(0).getEncodedName().equals(region2.getEncodedName()) ? mergedRegions.get(1) : mergedRegions.get(0);
        // snapshot
        admin.snapshot(snapshotName, tableName);
        Assert.assertEquals(1, admin.listSnapshots().size());
        // major compact
        admin.compactionSwitch(true, serverList);
        admin.majorCompactRegion(mergedRegion.getRegionName());
        // wait until merged region has no reference
        UTIL.waitFor(timeout, () -> {
            try {
                for (RegionServerThread regionServerThread : UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
                    HRegionServer regionServer = regionServerThread.getRegionServer();
                    for (HRegion subRegion : regionServer.getRegions(tableName)) {
                        if (subRegion.getRegionInfo().getEncodedName().equals(mergedRegion.getEncodedName())) {
                            regionServer.getCompactedHFilesDischarger().chore();
                        }
                    }
                }
                Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
                HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(UTIL.getConfiguration(), fs, tableDir, mergedRegion, true);
                return !regionFs.hasReferences(admin.getDescriptor(tableName));
            } catch (IOException e) {
                LOG.warn("Failed check merged region has no reference", e);
                return false;
            }
        });
        // run catalog janitor to clean and wait for parent regions are archived
        UTIL.getMiniHBaseCluster().getMaster().getCatalogJanitor().choreForTesting();
        UTIL.waitFor(timeout, () -> {
            try {
                Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
                for (FileStatus fileStatus : fs.listStatus(tableDir)) {
                    String name = fileStatus.getPath().getName();
                    if (name.equals(region0.getEncodedName()) || name.equals(region1.getEncodedName())) {
                        return false;
                    }
                }
                return true;
            } catch (IOException e) {
                LOG.warn("Check if parent regions are archived error", e);
                return false;
            }
        });
        // set file modify time and then run cleaner
        long time = EnvironmentEdgeManager.currentTime() - TimeToLiveHFileCleaner.DEFAULT_TTL * 1000;
        traverseAndSetFileTime(HFileArchiveUtil.getArchivePath(conf), time);
        UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().runCleaner();
        // scan snapshot
        try (TableSnapshotScanner scanner = new TableSnapshotScanner(conf, UTIL.getDataTestDirOnTestFS(snapshotName), snapshotName, new Scan().withStartRow(bbb).withStopRow(yyy))) {
            verifyScanner(scanner, bbb, yyy);
        }
    } catch (Exception e) {
        LOG.error("scan snapshot error", e);
        Assert.fail("Should not throw Exception: " + e.getMessage());
    } finally {
        tearDownCluster();
    }
}

Example 15 with HRegionFileSystem

use of org.apache.hadoop.hbase.regionserver.HRegionFileSystem in project hbase by apache.

the class SnapshotManifest method addRegion.

protected void addRegion(Path tableDir, RegionInfo regionInfo, RegionVisitor visitor) throws IOException {
    boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo);
    try {
        Path baseDir = tableDir;
        // Open the RegionFS
        if (isMobRegion) {
            baseDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), regionInfo.getTable());
        }
        HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, rootFs, baseDir, regionInfo, true);
        monitor.rethrowException();
        // 1. dump region meta info into the snapshot directory
        LOG.debug("Storing region-info for snapshot.");
        Object regionData = visitor.regionOpen(regionInfo);
        monitor.rethrowException();
        // 2. iterate through all the stores in the region
        LOG.debug("Creating references for hfiles");
        // 1000 files/batch, far more than the number of store files under a single column family.
        for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) {
            Object familyData = visitor.familyOpen(regionData, cfd.getName());
            monitor.rethrowException();
            StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, htd, cfd, regionFs);
            List<StoreFileInfo> storeFiles = tracker.load();
            if (storeFiles.isEmpty()) {
                LOG.debug("No files under family: {}", cfd.getNameAsString());
                continue;
            }
            // 2.1. build the snapshot reference for the store
            // iterate through all the store's files and create "references".
            addReferenceFiles(visitor, regionData, familyData, storeFiles, false);
            visitor.familyClose(regionData, familyData);
        }
        visitor.regionClose(regionData);
    } catch (IOException e) {
        // the mob directory might not be created yet, so do nothing when it is a mob region
        if (!isMobRegion) {
            throw e;
        }
    }
}