Examples with HRegionLocation org.apache.hadoop.hbase.HRegionLocation used on opensource projects

Example 96 with HRegionLocation

use of org.apache.hadoop.hbase.HRegionLocation in project hbase by apache.

the class TableInputFormatBase method oneInputSplitPerRegion.

/**
 * Create one InputSplit per region
 *
 * @return The list of InputSplit for all the regions
 * @throws IOException throws IOException
 */
private List<InputSplit> oneInputSplitPerRegion() throws IOException {
    RegionSizeCalculator sizeCalculator = createRegionSizeCalculator(getRegionLocator(), getAdmin());
    TableName tableName = getTable().getName();
    Pair<byte[][], byte[][]> keys = getStartEndKeys();
    if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) {
        HRegionLocation regLoc = getRegionLocator().getRegionLocation(HConstants.EMPTY_BYTE_ARRAY, false);
        if (null == regLoc) {
            throw new IOException("Expecting at least one region.");
        }
        List<InputSplit> splits = new ArrayList<>(1);
        long regionSize = sizeCalculator.getRegionSize(regLoc.getRegion().getRegionName());
        // In the table input format for single table we do not need to
        // store the scan object in table split because it can be memory intensive and redundant
        // information to what is already stored in conf SCAN. See HBASE-25212
        TableSplit split = new TableSplit(tableName, null, HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, regLoc.getHostnamePort().split(Addressing.HOSTNAME_PORT_SEPARATOR)[0], regionSize);
        splits.add(split);
        return splits;
    }
    List<InputSplit> splits = new ArrayList<>(keys.getFirst().length);
    for (int i = 0; i < keys.getFirst().length; i++) {
        if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
            continue;
        }
        byte[] startRow = scan.getStartRow();
        byte[] stopRow = scan.getStopRow();
        // determine if the given start an stop key fall into the region
        if ((startRow.length == 0 || keys.getSecond()[i].length == 0 || Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) && (stopRow.length == 0 || Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) {
            byte[] splitStart = startRow.length == 0 || Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ? keys.getFirst()[i] : startRow;
            byte[] splitStop = (stopRow.length == 0 || Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0) && keys.getSecond()[i].length > 0 ? keys.getSecond()[i] : stopRow;
            HRegionLocation location = getRegionLocator().getRegionLocation(keys.getFirst()[i], false);
            // The below InetSocketAddress creation does a name resolution.
            InetSocketAddress isa = new InetSocketAddress(location.getHostname(), location.getPort());
            if (isa.isUnresolved()) {
                LOG.warn("Failed resolve " + isa);
            }
            InetAddress regionAddress = isa.getAddress();
            String regionLocation;
            regionLocation = reverseDNS(regionAddress);
            byte[] regionName = location.getRegion().getRegionName();
            String encodedRegionName = location.getRegion().getEncodedName();
            long regionSize = sizeCalculator.getRegionSize(regionName);
            // In the table input format for single table we do not need to
            // store the scan object in table split because it can be memory intensive and redundant
            // information to what is already stored in conf SCAN. See HBASE-25212
            TableSplit split = new TableSplit(tableName, null, splitStart, splitStop, regionLocation, encodedRegionName, regionSize);
            splits.add(split);
            if (LOG.isDebugEnabled()) {
                LOG.debug("getSplits: split -> " + i + " -> " + split);
            }
        }
    }
    return splits;
}

Example 97 with HRegionLocation

use of org.apache.hadoop.hbase.HRegionLocation in project hbase by apache.

the class TestCoprocessorEndpoint method testCoprocessorServiceNullResponse.

@Test
public void testCoprocessorServiceNullResponse() throws Throwable {
    Table table = util.getConnection().getTable(TEST_TABLE);
    List<HRegionLocation> regions;
    try (RegionLocator rl = util.getConnection().getRegionLocator(TEST_TABLE)) {
        regions = rl.getAllRegionLocations();
    }
    final TestProtos.EchoRequestProto request = TestProtos.EchoRequestProto.newBuilder().setMessage("hello").build();
    try {
        // scan: for all regions
        final RpcController controller = new ServerRpcController();
        // test that null results are supported
        Map<byte[], String> results = table.coprocessorService(TestRpcServiceProtos.TestProtobufRpcProto.class, ROWS[0], ROWS[ROWS.length - 1], new Batch.Call<TestRpcServiceProtos.TestProtobufRpcProto, String>() {

            public String call(TestRpcServiceProtos.TestProtobufRpcProto instance) throws IOException {
                CoprocessorRpcUtils.BlockingRpcCallback<TestProtos.EchoResponseProto> callback = new CoprocessorRpcUtils.BlockingRpcCallback<>();
                instance.echo(controller, request, callback);
                TestProtos.EchoResponseProto response = callback.get();
                LOG.debug("Batch.Call got result " + response);
                return null;
            }
        });
        for (Map.Entry<byte[], String> e : results.entrySet()) {
            LOG.info("Got value " + e.getValue() + " for region " + Bytes.toStringBinary(e.getKey()));
        }
        assertEquals(3, results.size());
        for (HRegionLocation region : regions) {
            RegionInfo info = region.getRegion();
            LOG.info("Region info is " + info.getRegionNameAsString());
            assertTrue(results.containsKey(info.getRegionName()));
            assertNull(results.get(info.getRegionName()));
        }
    } finally {
        table.close();
    }
}

Example 98 with HRegionLocation

use of org.apache.hadoop.hbase.HRegionLocation in project hbase by apache.

the class TestCoprocessorEndpoint method testCoprocessorService.

@Test
public void testCoprocessorService() throws Throwable {
    Table table = util.getConnection().getTable(TEST_TABLE);
    List<HRegionLocation> regions;
    try (RegionLocator rl = util.getConnection().getRegionLocator(TEST_TABLE)) {
        regions = rl.getAllRegionLocations();
    }
    final TestProtos.EchoRequestProto request = TestProtos.EchoRequestProto.newBuilder().setMessage("hello").build();
    final Map<byte[], String> results = Collections.synchronizedMap(new TreeMap<byte[], String>(Bytes.BYTES_COMPARATOR));
    try {
        // scan: for all regions
        final RpcController controller = new ServerRpcController();
        table.coprocessorService(TestRpcServiceProtos.TestProtobufRpcProto.class, ROWS[0], ROWS[ROWS.length - 1], new Batch.Call<TestRpcServiceProtos.TestProtobufRpcProto, TestProtos.EchoResponseProto>() {

            @Override
            public TestProtos.EchoResponseProto call(TestRpcServiceProtos.TestProtobufRpcProto instance) throws IOException {
                LOG.debug("Default response is " + TestProtos.EchoRequestProto.getDefaultInstance());
                CoprocessorRpcUtils.BlockingRpcCallback<TestProtos.EchoResponseProto> callback = new CoprocessorRpcUtils.BlockingRpcCallback<>();
                instance.echo(controller, request, callback);
                TestProtos.EchoResponseProto response = callback.get();
                LOG.debug("Batch.Call returning result " + response);
                return response;
            }
        }, new Batch.Callback<TestProtos.EchoResponseProto>() {

            @Override
            public void update(byte[] region, byte[] row, TestProtos.EchoResponseProto result) {
                assertNotNull(result);
                assertEquals("hello", result.getMessage());
                results.put(region, result.getMessage());
            }
        });
        for (Map.Entry<byte[], String> e : results.entrySet()) {
            LOG.info("Got value " + e.getValue() + " for region " + Bytes.toStringBinary(e.getKey()));
        }
        assertEquals(3, results.size());
        for (HRegionLocation info : regions) {
            LOG.info("Region info is " + info.getRegion().getRegionNameAsString());
            assertTrue(results.containsKey(info.getRegion().getRegionName()));
        }
        results.clear();
        // scan: for region 2 and region 3
        table.coprocessorService(TestRpcServiceProtos.TestProtobufRpcProto.class, ROWS[rowSeperator1], ROWS[ROWS.length - 1], new Batch.Call<TestRpcServiceProtos.TestProtobufRpcProto, TestProtos.EchoResponseProto>() {

            @Override
            public TestProtos.EchoResponseProto call(TestRpcServiceProtos.TestProtobufRpcProto instance) throws IOException {
                LOG.debug("Default response is " + TestProtos.EchoRequestProto.getDefaultInstance());
                CoprocessorRpcUtils.BlockingRpcCallback<TestProtos.EchoResponseProto> callback = new CoprocessorRpcUtils.BlockingRpcCallback<>();
                instance.echo(controller, request, callback);
                TestProtos.EchoResponseProto response = callback.get();
                LOG.debug("Batch.Call returning result " + response);
                return response;
            }
        }, new Batch.Callback<TestProtos.EchoResponseProto>() {

            @Override
            public void update(byte[] region, byte[] row, TestProtos.EchoResponseProto result) {
                assertNotNull(result);
                assertEquals("hello", result.getMessage());
                results.put(region, result.getMessage());
            }
        });
        for (Map.Entry<byte[], String> e : results.entrySet()) {
            LOG.info("Got value " + e.getValue() + " for region " + Bytes.toStringBinary(e.getKey()));
        }
        assertEquals(2, results.size());
    } finally {
        table.close();
    }
}

Example 99 with HRegionLocation

use of org.apache.hadoop.hbase.HRegionLocation in project hbase by apache.

the class RegionsResource method get.

@GET
@Produces({ MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, MIMETYPE_PROTOBUF_IETF })
public Response get(@Context final UriInfo uriInfo) {
    if (LOG.isTraceEnabled()) {
        LOG.trace("GET " + uriInfo.getAbsolutePath());
    }
    servlet.getMetrics().incrementRequests(1);
    try {
        TableName tableName = TableName.valueOf(tableResource.getName());
        if (!tableResource.exists()) {
            throw new TableNotFoundException(tableName);
        }
        TableInfoModel model = new TableInfoModel(tableName.getNameAsString());
        List<HRegionLocation> locs;
        try (Connection connection = ConnectionFactory.createConnection(servlet.getConfiguration());
            RegionLocator locator = connection.getRegionLocator(tableName)) {
            locs = locator.getAllRegionLocations();
        }
        for (HRegionLocation loc : locs) {
            RegionInfo hri = loc.getRegion();
            ServerName addr = loc.getServerName();
            model.add(new TableRegionModel(tableName.getNameAsString(), hri.getRegionId(), hri.getStartKey(), hri.getEndKey(), addr.getAddress().toString()));
        }
        ResponseBuilder response = Response.ok(model);
        response.cacheControl(cacheControl);
        servlet.getMetrics().incrementSucessfulGetRequests(1);
        return response.build();
    } catch (TableNotFoundException e) {
        servlet.getMetrics().incrementFailedGetRequests(1);
        return Response.status(Response.Status.NOT_FOUND).type(MIMETYPE_TEXT).entity("Not found" + CRLF).build();
    } catch (IOException e) {
        servlet.getMetrics().incrementFailedGetRequests(1);
        return Response.status(Response.Status.SERVICE_UNAVAILABLE).type(MIMETYPE_TEXT).entity("Unavailable" + CRLF).build();
    }
}

Example 100 with HRegionLocation

use of org.apache.hadoop.hbase.HRegionLocation in project hbase by apache.

the class TableSnapshotInputFormatImpl method getSplits.

public static List<InputSplit> getSplits(Scan scan, SnapshotManifest manifest, List<RegionInfo> regionManifests, Path restoreDir, Configuration conf, RegionSplitter.SplitAlgorithm sa, int numSplits) throws IOException {
    // load table descriptor
    TableDescriptor htd = manifest.getTableDescriptor();
    Path tableDir = CommonFSUtils.getTableDir(restoreDir, htd.getTableName());
    boolean localityEnabled = conf.getBoolean(SNAPSHOT_INPUTFORMAT_LOCALITY_ENABLED_KEY, SNAPSHOT_INPUTFORMAT_LOCALITY_ENABLED_DEFAULT);
    boolean scanMetricsEnabled = conf.getBoolean(SNAPSHOT_INPUTFORMAT_SCAN_METRICS_ENABLED, SNAPSHOT_INPUTFORMAT_SCAN_METRICS_ENABLED_DEFAULT);
    scan.setScanMetricsEnabled(scanMetricsEnabled);
    boolean useRegionLoc = conf.getBoolean(SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION, SNAPSHOT_INPUTFORMAT_LOCALITY_BY_REGION_LOCATION_DEFAULT);
    Connection connection = null;
    RegionLocator regionLocator = null;
    if (localityEnabled && useRegionLoc) {
        Configuration newConf = new Configuration(conf);
        newConf.setInt("hbase.hconnection.threads.max", 1);
        try {
            connection = ConnectionFactory.createConnection(newConf);
            regionLocator = connection.getRegionLocator(htd.getTableName());
            /* Get all locations for the table and cache it */
            regionLocator.getAllRegionLocations();
        } finally {
            if (connection != null) {
                connection.close();
            }
        }
    }
    List<InputSplit> splits = new ArrayList<>();
    for (RegionInfo hri : regionManifests) {
        // load region descriptor
        List<String> hosts = null;
        if (localityEnabled) {
            if (regionLocator != null) {
                /* Get Location from the local cache */
                HRegionLocation location = regionLocator.getRegionLocation(hri.getStartKey(), false);
                hosts = new ArrayList<>(1);
                hosts.add(location.getHostname());
            } else {
                hosts = calculateLocationsForInputSplit(conf, htd, hri, tableDir);
            }
        }
        if (numSplits > 1) {
            byte[][] sp = sa.split(hri.getStartKey(), hri.getEndKey(), numSplits, true);
            for (int i = 0; i < sp.length - 1; i++) {
                if (PrivateCellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), sp[i], sp[i + 1])) {
                    Scan boundedScan = new Scan(scan);
                    if (scan.getStartRow().length == 0) {
                        boundedScan.withStartRow(sp[i]);
                    } else {
                        boundedScan.withStartRow(Bytes.compareTo(scan.getStartRow(), sp[i]) > 0 ? scan.getStartRow() : sp[i]);
                    }
                    if (scan.getStopRow().length == 0) {
                        boundedScan.withStopRow(sp[i + 1]);
                    } else {
                        boundedScan.withStopRow(Bytes.compareTo(scan.getStopRow(), sp[i + 1]) < 0 ? scan.getStopRow() : sp[i + 1]);
                    }
                    splits.add(new InputSplit(htd, hri, hosts, boundedScan, restoreDir));
                }
            }
        } else {
            if (PrivateCellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), hri.getStartKey(), hri.getEndKey())) {
                splits.add(new InputSplit(htd, hri, hosts, scan, restoreDir));
            }
        }
    }
    return splits;
}