Examples with RegionLocator org.apache.hadoop.hbase.client.RegionLocator used on opensource projects

Example 6 with RegionLocator

use of org.apache.hadoop.hbase.client.RegionLocator in project hbase by apache.

the class RegionSplitter method splitScan.

static LinkedList<Pair<byte[], byte[]>> splitScan(LinkedList<Pair<byte[], byte[]>> regionList, final Connection connection, final TableName tableName, SplitAlgorithm splitAlgo) throws IOException, InterruptedException {
    LinkedList<Pair<byte[], byte[]>> finished = Lists.newLinkedList();
    LinkedList<Pair<byte[], byte[]>> logicalSplitting = Lists.newLinkedList();
    LinkedList<Pair<byte[], byte[]>> physicalSplitting = Lists.newLinkedList();
    // Get table info
    Pair<Path, Path> tableDirAndSplitFile = getTableDirAndSplitFile(connection.getConfiguration(), tableName);
    Path tableDir = tableDirAndSplitFile.getFirst();
    FileSystem fs = tableDir.getFileSystem(connection.getConfiguration());
    // Clear the cache to forcibly refresh region information
    ((ClusterConnection) connection).clearRegionCache();
    HTableDescriptor htd = null;
    try (Table table = connection.getTable(tableName)) {
        htd = table.getTableDescriptor();
    }
    try (RegionLocator regionLocator = connection.getRegionLocator(tableName)) {
        // for every region that hasn't been verified as a finished split
        for (Pair<byte[], byte[]> region : regionList) {
            byte[] start = region.getFirst();
            byte[] split = region.getSecond();
            // see if the new split daughter region has come online
            try {
                HRegionInfo dri = regionLocator.getRegionLocation(split).getRegionInfo();
                if (dri.isOffline() || !Bytes.equals(dri.getStartKey(), split)) {
                    logicalSplitting.add(region);
                    continue;
                }
            } catch (NoServerForRegionException nsfre) {
                // NSFRE will occur if the old hbase:meta entry has no server assigned
                LOG.info(nsfre);
                logicalSplitting.add(region);
                continue;
            }
            try {
                // when a daughter region is opened, a compaction is triggered
                // wait until compaction completes for both daughter regions
                LinkedList<HRegionInfo> check = Lists.newLinkedList();
                check.add(regionLocator.getRegionLocation(start).getRegionInfo());
                check.add(regionLocator.getRegionLocation(split).getRegionInfo());
                for (HRegionInfo hri : check.toArray(new HRegionInfo[check.size()])) {
                    byte[] sk = hri.getStartKey();
                    if (sk.length == 0)
                        sk = splitAlgo.firstRow();
                    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(connection.getConfiguration(), fs, tableDir, hri, true);
                    // Check every Column Family for that region -- check does not have references.
                    boolean refFound = false;
                    for (HColumnDescriptor c : htd.getFamilies()) {
                        if ((refFound = regionFs.hasReferences(c.getNameAsString()))) {
                            break;
                        }
                    }
                    // compaction is completed when all reference files are gone
                    if (!refFound) {
                        check.remove(hri);
                    }
                }
                if (check.isEmpty()) {
                    finished.add(region);
                } else {
                    physicalSplitting.add(region);
                }
            } catch (NoServerForRegionException nsfre) {
                LOG.debug("No Server Exception thrown for: " + splitAlgo.rowToStr(start));
                physicalSplitting.add(region);
                ((ClusterConnection) connection).clearRegionCache();
            }
        }
        LOG.debug("Split Scan: " + finished.size() + " finished / " + logicalSplitting.size() + " split wait / " + physicalSplitting.size() + " reference wait");
        return finished;
    }
}

Example 7 with RegionLocator

use of org.apache.hadoop.hbase.client.RegionLocator in project hbase by apache.

the class IntegrationTestBulkLoad method runLinkedListMRJob.

private void runLinkedListMRJob(int iteration) throws Exception {
    String jobName = IntegrationTestBulkLoad.class.getSimpleName() + " - " + EnvironmentEdgeManager.currentTime();
    Configuration conf = new Configuration(util.getConfiguration());
    Path p = null;
    if (conf.get(ImportTsv.BULK_OUTPUT_CONF_KEY) == null) {
        p = util.getDataTestDirOnTestFS(getTablename() + "-" + iteration);
    } else {
        p = new Path(conf.get(ImportTsv.BULK_OUTPUT_CONF_KEY));
    }
    conf.setBoolean("mapreduce.map.speculative", false);
    conf.setBoolean("mapreduce.reduce.speculative", false);
    conf.setInt(ROUND_NUM_KEY, iteration);
    Job job = new Job(conf);
    job.setJobName(jobName);
    // set the input format so that we can create map tasks with no data input.
    job.setInputFormatClass(ITBulkLoadInputFormat.class);
    // Set the mapper classes.
    job.setMapperClass(LinkedListCreationMapper.class);
    job.setMapOutputKeyClass(ImmutableBytesWritable.class);
    job.setMapOutputValueClass(KeyValue.class);
    // Use the identity reducer
    // So nothing to do here.
    // Set this jar.
    job.setJarByClass(getClass());
    // Set where to place the hfiles.
    FileOutputFormat.setOutputPath(job, p);
    try (Connection conn = ConnectionFactory.createConnection(conf);
        Admin admin = conn.getAdmin();
        Table table = conn.getTable(getTablename());
        RegionLocator regionLocator = conn.getRegionLocator(getTablename())) {
        // Configure the partitioner and other things needed for HFileOutputFormat.
        HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
        // Run the job making sure it works.
        assertEquals(true, job.waitForCompletion(true));
        // Create a new loader.
        LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
        // Load the HFiles in.
        loader.doBulkLoad(p, admin, table, regionLocator);
    }
    // Delete the files.
    util.getTestFileSystem().delete(p, true);
}

Example 8 with RegionLocator

use of org.apache.hadoop.hbase.client.RegionLocator in project hbase by apache.

the class Canary method sniff.

/*
   * Loops over regions that owns this table, and output some information abouts the state.
   */
private static List<Future<Void>> sniff(final Admin admin, final Sink sink, HTableDescriptor tableDesc, ExecutorService executor, TaskType taskType, boolean rawScanEnabled) throws Exception {
    if (LOG.isDebugEnabled()) {
        LOG.debug(String.format("reading list of regions for table %s", tableDesc.getTableName()));
    }
    Table table = null;
    try {
        table = admin.getConnection().getTable(tableDesc.getTableName());
    } catch (TableNotFoundException e) {
        return new ArrayList<>();
    } finally {
        if (table != null) {
            table.close();
        }
    }
    List<RegionTask> tasks = new ArrayList<>();
    RegionLocator regionLocator = null;
    try {
        regionLocator = admin.getConnection().getRegionLocator(tableDesc.getTableName());
        for (HRegionLocation location : regionLocator.getAllRegionLocations()) {
            ServerName rs = location.getServerName();
            HRegionInfo region = location.getRegionInfo();
            tasks.add(new RegionTask(admin.getConnection(), region, rs, sink, taskType, rawScanEnabled));
        }
    } finally {
        if (regionLocator != null) {
            regionLocator.close();
        }
    }
    return executor.invokeAll(tasks);
}

Example 9 with RegionLocator

use of org.apache.hadoop.hbase.client.RegionLocator in project hbase by apache.

the class HFileSplitterJob method createSubmittableJob.

/**
   * Sets up the actual job.
   * @param args The command line parameters.
   * @return The newly created job.
   * @throws IOException When setting up the job fails.
   */
public Job createSubmittableJob(String[] args) throws IOException {
    Configuration conf = getConf();
    String inputDirs = args[0];
    String tabName = args[1];
    conf.setStrings(TABLES_KEY, tabName);
    Job job = Job.getInstance(conf, conf.get(JOB_NAME_CONF_KEY, NAME + "_" + EnvironmentEdgeManager.currentTime()));
    job.setJarByClass(HFileSplitterJob.class);
    FileInputFormat.addInputPaths(job, inputDirs);
    job.setInputFormatClass(HFileInputFormat.class);
    job.setMapOutputKeyClass(ImmutableBytesWritable.class);
    String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY);
    if (hfileOutPath != null) {
        LOG.debug("add incremental job :" + hfileOutPath + " from " + inputDirs);
        TableName tableName = TableName.valueOf(tabName);
        job.setMapperClass(HFileCellMapper.class);
        job.setReducerClass(KeyValueSortReducer.class);
        Path outputDir = new Path(hfileOutPath);
        FileOutputFormat.setOutputPath(job, outputDir);
        job.setMapOutputValueClass(KeyValue.class);
        try (Connection conn = ConnectionFactory.createConnection(conf);
            Table table = conn.getTable(tableName);
            RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
            HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
        }
        LOG.debug("success configuring load incremental job");
        TableMapReduceUtil.addDependencyJars(job.getConfiguration(), com.google.common.base.Preconditions.class);
    } else {
        throw new IOException("No bulk output directory specified");
    }
    return job;
}

Example 10 with RegionLocator

use of org.apache.hadoop.hbase.client.RegionLocator in project hbase by apache.

the class TestMetaTableAccessor method testMetaUpdatesGoToPriorityQueue.

@Test
public void testMetaUpdatesGoToPriorityQueue() throws Exception {
    // This test has to be end-to-end, and do the verification from the server side
    Configuration c = UTIL.getConfiguration();
    c.set(RSRpcServices.REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS, SpyingRpcSchedulerFactory.class.getName());
    // restart so that new config takes place
    afterClass();
    beforeClass();
    final TableName tableName = TableName.valueOf(name.getMethodName());
    try (Admin admin = connection.getAdmin();
        RegionLocator rl = connection.getRegionLocator(tableName)) {
        // create a table and prepare for a manual split
        UTIL.createTable(tableName, "cf1");
        HRegionLocation loc = rl.getAllRegionLocations().get(0);
        HRegionInfo parent = loc.getRegionInfo();
        long rid = 1000;
        byte[] splitKey = Bytes.toBytes("a");
        HRegionInfo splitA = new HRegionInfo(parent.getTable(), parent.getStartKey(), splitKey, false, rid);
        HRegionInfo splitB = new HRegionInfo(parent.getTable(), splitKey, parent.getEndKey(), false, rid);
        // find the meta server
        MiniHBaseCluster cluster = UTIL.getMiniHBaseCluster();
        int rsIndex = cluster.getServerWithMeta();
        HRegionServer rs;
        if (rsIndex >= 0) {
            rs = cluster.getRegionServer(rsIndex);
        } else {
            // it is in master
            rs = cluster.getMaster();
        }
        SpyingRpcScheduler scheduler = (SpyingRpcScheduler) rs.getRpcServer().getScheduler();
        long prevCalls = scheduler.numPriorityCalls;
        MetaTableAccessor.splitRegion(connection, parent, splitA, splitB, loc.getServerName(), 1, false);
        assertTrue(prevCalls < scheduler.numPriorityCalls);
    }
}