Examples with FSTableDescriptors org.apache.hadoop.hbase.util.FSTableDescriptors used on opensource projects

Example 11 with FSTableDescriptors

use of org.apache.hadoop.hbase.util.FSTableDescriptors in project hbase by apache.

the class HBaseServerBase method initializeFileSystem.

protected final void initializeFileSystem() throws IOException {
    // Get fs instance used by this RS. Do we use checksum verification in the hbase? If hbase
    // checksum verification enabled, then automatically switch off hdfs checksum verification.
    boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true);
    String walDirUri = CommonFSUtils.getDirUri(this.conf, new Path(conf.get(CommonFSUtils.HBASE_WAL_DIR, conf.get(HConstants.HBASE_DIR))));
    // set WAL's uri
    if (walDirUri != null) {
        CommonFSUtils.setFsDefault(this.conf, walDirUri);
    }
    // init the WALFs
    this.walFs = new HFileSystem(this.conf, useHBaseChecksum);
    this.walRootDir = CommonFSUtils.getWALRootDir(this.conf);
    // Set 'fs.defaultFS' to match the filesystem on hbase.rootdir else
    // underlying hadoop hdfs accessors will be going against wrong filesystem
    // (unless all is set to defaults).
    String rootDirUri = CommonFSUtils.getDirUri(this.conf, new Path(conf.get(HConstants.HBASE_DIR)));
    if (rootDirUri != null) {
        CommonFSUtils.setFsDefault(this.conf, rootDirUri);
    }
    // init the filesystem
    this.dataFs = new HFileSystem(this.conf, useHBaseChecksum);
    this.dataRootDir = CommonFSUtils.getRootDir(this.conf);
    this.tableDescriptors = new FSTableDescriptors(this.dataFs, this.dataRootDir, !canUpdateTableDescriptor(), cacheTableDescriptor());
}

Example 12 with FSTableDescriptors

use of org.apache.hadoop.hbase.util.FSTableDescriptors in project hbase by apache.

the class CloneSnapshotProcedure method createFsLayout.

/**
   * Create region layout in file system.
   * @param env MasterProcedureEnv
   * @throws IOException
   */
private List<HRegionInfo> createFsLayout(final MasterProcedureEnv env, final HTableDescriptor hTableDescriptor, List<HRegionInfo> newRegions, final CreateHdfsRegions hdfsRegionHandler) throws IOException {
    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
    final Path tempdir = mfs.getTempDir();
    // 1. Create Table Descriptor
    // using a copy of descriptor, table will be created enabling first
    HTableDescriptor underConstruction = new HTableDescriptor(hTableDescriptor);
    final Path tempTableDir = FSUtils.getTableDir(tempdir, hTableDescriptor.getTableName());
    ((FSTableDescriptors) (env.getMasterServices().getTableDescriptors())).createTableDescriptorForTableDirectory(tempTableDir, underConstruction, false);
    // 2. Create Regions
    newRegions = hdfsRegionHandler.createHdfsRegions(env, tempdir, hTableDescriptor.getTableName(), newRegions);
    // 3. Move Table temp directory to the hbase root location
    CreateTableProcedure.moveTempDirectoryToHBaseRoot(env, hTableDescriptor, tempTableDir);
    return newRegions;
}

Example 13 with FSTableDescriptors

use of org.apache.hadoop.hbase.util.FSTableDescriptors in project hbase by apache.

the class CreateTableProcedure method createFsLayout.

protected static List<HRegionInfo> createFsLayout(final MasterProcedureEnv env, final HTableDescriptor hTableDescriptor, List<HRegionInfo> newRegions, final CreateHdfsRegions hdfsRegionHandler) throws IOException {
    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
    final Path tempdir = mfs.getTempDir();
    // 1. Create Table Descriptor
    // using a copy of descriptor, table will be created enabling first
    final Path tempTableDir = FSUtils.getTableDir(tempdir, hTableDescriptor.getTableName());
    ((FSTableDescriptors) (env.getMasterServices().getTableDescriptors())).createTableDescriptorForTableDirectory(tempTableDir, hTableDescriptor, false);
    // 2. Create Regions
    newRegions = hdfsRegionHandler.createHdfsRegions(env, tempdir, hTableDescriptor.getTableName(), newRegions);
    // 3. Move Table temp directory to the hbase root location
    moveTempDirectoryToHBaseRoot(env, hTableDescriptor, tempTableDir);
    return newRegions;
}

Example 14 with FSTableDescriptors

use of org.apache.hadoop.hbase.util.FSTableDescriptors in project hbase by apache.

the class TestMasterFailover method testPendingOpenOrCloseWhenMasterFailover.

/**
   * Test region in pending_open/close when master failover
   */
@Test(timeout = 180000)
public void testPendingOpenOrCloseWhenMasterFailover() throws Exception {
    final int NUM_MASTERS = 1;
    final int NUM_RS = 1;
    // Create config to use for this cluster
    Configuration conf = HBaseConfiguration.create();
    // Start the cluster
    HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
    TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
    log("Cluster started");
    // get all the master threads
    List<MasterThread> masterThreads = cluster.getMasterThreads();
    assertEquals(1, masterThreads.size());
    // only one master thread, let's wait for it to be initialized
    assertTrue(cluster.waitForActiveAndReadyMaster());
    HMaster master = masterThreads.get(0).getMaster();
    assertTrue(master.isActiveMaster());
    assertTrue(master.isInitialized());
    // Create a table with a region online
    Table onlineTable = TEST_UTIL.createTable(TableName.valueOf("onlineTable"), "family");
    onlineTable.close();
    // Create a table in META, so it has a region offline
    HTableDescriptor offlineTable = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("offlineTable")));
    offlineTable.addFamily(new HColumnDescriptor(Bytes.toBytes("family")));
    FileSystem filesystem = FileSystem.get(conf);
    Path rootdir = FSUtils.getRootDir(conf);
    FSTableDescriptors fstd = new FSTableDescriptors(conf, filesystem, rootdir);
    fstd.createTableDescriptor(offlineTable);
    HRegionInfo hriOffline = new HRegionInfo(offlineTable.getTableName(), null, null);
    createRegion(hriOffline, rootdir, conf, offlineTable);
    MetaTableAccessor.addRegionToMeta(master.getConnection(), hriOffline);
    log("Regions in hbase:meta and namespace have been created");
    // at this point we only expect 3 regions to be assigned out
    // (catalogs and namespace, + 1 online region)
    assertEquals(3, cluster.countServedRegions());
    HRegionInfo hriOnline = null;
    try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(TableName.valueOf("onlineTable"))) {
        hriOnline = locator.getRegionLocation(HConstants.EMPTY_START_ROW).getRegionInfo();
    }
    RegionStates regionStates = master.getAssignmentManager().getRegionStates();
    RegionStateStore stateStore = master.getAssignmentManager().getRegionStateStore();
    // Put the online region in pending_close. It is actually already opened.
    // This is to simulate that the region close RPC is not sent out before failover
    RegionState oldState = regionStates.getRegionState(hriOnline);
    RegionState newState = new RegionState(hriOnline, State.PENDING_CLOSE, oldState.getServerName());
    stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
    // Put the offline region in pending_open. It is actually not opened yet.
    // This is to simulate that the region open RPC is not sent out before failover
    oldState = new RegionState(hriOffline, State.OFFLINE);
    newState = new RegionState(hriOffline, State.PENDING_OPEN, newState.getServerName());
    stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
    HRegionInfo failedClose = new HRegionInfo(offlineTable.getTableName(), null, null);
    createRegion(failedClose, rootdir, conf, offlineTable);
    MetaTableAccessor.addRegionToMeta(master.getConnection(), failedClose);
    oldState = new RegionState(failedClose, State.PENDING_CLOSE);
    newState = new RegionState(failedClose, State.FAILED_CLOSE, newState.getServerName());
    stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
    HRegionInfo failedOpen = new HRegionInfo(offlineTable.getTableName(), null, null);
    createRegion(failedOpen, rootdir, conf, offlineTable);
    MetaTableAccessor.addRegionToMeta(master.getConnection(), failedOpen);
    // Simulate a region transitioning to failed open when the region server reports the
    // transition as FAILED_OPEN
    oldState = new RegionState(failedOpen, State.PENDING_OPEN);
    newState = new RegionState(failedOpen, State.FAILED_OPEN, newState.getServerName());
    stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
    HRegionInfo failedOpenNullServer = new HRegionInfo(offlineTable.getTableName(), null, null);
    LOG.info("Failed open NUll server " + failedOpenNullServer.getEncodedName());
    createRegion(failedOpenNullServer, rootdir, conf, offlineTable);
    MetaTableAccessor.addRegionToMeta(master.getConnection(), failedOpenNullServer);
    // Simulate a region transitioning to failed open when the master couldn't find a plan for
    // the region
    oldState = new RegionState(failedOpenNullServer, State.OFFLINE);
    newState = new RegionState(failedOpenNullServer, State.FAILED_OPEN, null);
    stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
    // Stop the master
    log("Aborting master");
    cluster.abortMaster(0);
    cluster.waitOnMaster(0);
    log("Master has aborted");
    // Start up a new master
    log("Starting up a new master");
    master = cluster.startMaster().getMaster();
    log("Waiting for master to be ready");
    cluster.waitForActiveAndReadyMaster();
    log("Master is ready");
    // Wait till no region in transition any more
    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
    // Get new region states since master restarted
    regionStates = master.getAssignmentManager().getRegionStates();
    // Both pending_open (RPC sent/not yet) regions should be online
    assertTrue(regionStates.isRegionOnline(hriOffline));
    assertTrue(regionStates.isRegionOnline(hriOnline));
    assertTrue(regionStates.isRegionOnline(failedClose));
    assertTrue(regionStates.isRegionOnline(failedOpenNullServer));
    assertTrue(regionStates.isRegionOnline(failedOpen));
    log("Done with verification, shutting down cluster");
    // Done, shutdown the cluster
    TEST_UTIL.shutdownMiniCluster();
}

Example 15 with FSTableDescriptors

use of org.apache.hadoop.hbase.util.FSTableDescriptors in project hbase by apache.

the class TestRestoreSnapshotHelper method testRestore.

/**
   * Execute the restore operation
   * @param snapshotDir The snapshot directory to use as "restore source"
   * @param sd The snapshot descriptor
   * @param htdClone The HTableDescriptor of the table to restore/clone.
   */
private void testRestore(final Path snapshotDir, final SnapshotDescription sd, final HTableDescriptor htdClone) throws IOException {
    LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
    FSUtils.logFileSystemState(fs, rootDir, LOG);
    new FSTableDescriptors(conf).createTableDescriptor(htdClone);
    RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone);
    helper.restoreHdfsRegions();
    LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
    FSUtils.logFileSystemState(fs, rootDir, LOG);
}