Examples with HMaster org.apache.hadoop.hbase.master.HMaster used on opensource projects

Example 56 with HMaster

use of org.apache.hadoop.hbase.master.HMaster in project hbase by apache.

the class TestRegionSizeUse method testBasicRegionSizeReports.

@Test
public void testBasicRegionSizeReports() throws Exception {
    // 5MB
    final long bytesWritten = 5L * 1024L * 1024L;
    final TableName tn = writeData(bytesWritten);
    LOG.debug("Data was written to HBase");
    final Admin admin = TEST_UTIL.getAdmin();
    // Push the data to disk.
    admin.flush(tn);
    LOG.debug("Data flushed to disk");
    // Get the final region distribution
    final List<RegionInfo> regions = TEST_UTIL.getAdmin().getRegions(tn);
    HMaster master = cluster.getMaster();
    MasterQuotaManager quotaManager = master.getMasterQuotaManager();
    Map<RegionInfo, Long> regionSizes = quotaManager.snapshotRegionSizes();
    // Wait until we get all of the region reports for our table
    // The table may split, so make sure we have at least as many as expected right after we
    // finished writing the data.
    int observedRegions = numRegionsForTable(tn, regionSizes);
    while (observedRegions < regions.size()) {
        LOG.debug("Expecting more regions. Saw " + observedRegions + " region sizes reported, expected at least " + regions.size());
        Thread.sleep(1000);
        regionSizes = quotaManager.snapshotRegionSizes();
        observedRegions = numRegionsForTable(tn, regionSizes);
    }
    LOG.debug("Observed region sizes by the HMaster: " + regionSizes);
    long totalRegionSize = 0L;
    for (Long regionSize : regionSizes.values()) {
        totalRegionSize += regionSize;
    }
    assertTrue("Expected region size report to exceed " + bytesWritten + ", but was " + totalRegionSize + ". RegionSizes=" + regionSizes, bytesWritten < totalRegionSize);
}

Example 57 with HMaster

use of org.apache.hadoop.hbase.master.HMaster in project hbase by apache.

the class TestQuotaObserverChoreRegionReports method testMissingReportsRemovesQuota.

@Test
public void testMissingReportsRemovesQuota() throws Exception {
    Configuration conf = TEST_UTIL.getConfiguration();
    // Expire the reports after 5 seconds
    conf.setInt(QuotaObserverChore.REGION_REPORT_RETENTION_DURATION_KEY, 5000);
    TEST_UTIL.startMiniCluster(1);
    // Wait till quota table onlined.
    TEST_UTIL.waitFor(10000, new Waiter.Predicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            return TEST_UTIL.getAdmin().tableExists(QuotaTableUtil.QUOTA_TABLE_NAME);
        }
    });
    final String FAM1 = "f1";
    // Create a table
    final TableName tn = TableName.valueOf("quotaAcceptanceWithoutReports");
    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tn).setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAM1)).build();
    TEST_UTIL.getAdmin().createTable(tableDesc);
    // Set a quota
    final long sizeLimit = 1L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
    final SpaceViolationPolicy violationPolicy = SpaceViolationPolicy.NO_INSERTS;
    QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, violationPolicy);
    final Admin admin = TEST_UTIL.getAdmin();
    LOG.info("SET QUOTA");
    admin.setQuota(settings);
    final Connection conn = TEST_UTIL.getConnection();
    // Write enough data to invalidate the quota
    Put p = new Put(Bytes.toBytes("row1"));
    byte[] bytes = new byte[10];
    Arrays.fill(bytes, (byte) 2);
    for (int i = 0; i < 200; i++) {
        p.addColumn(Bytes.toBytes(FAM1), Bytes.toBytes("qual" + i), bytes);
    }
    conn.getTable(tn).put(p);
    admin.flush(tn);
    // Wait for the table to move into violation
    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30000, 1000, new Waiter.Predicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            SpaceQuotaSnapshot snapshot = getSnapshotForTable(conn, tn);
            if (snapshot == null) {
                return false;
            }
            return snapshot.getQuotaStatus().isInViolation();
        }
    });
    // Close the region, prevent the server from sending new status reports.
    List<RegionInfo> regions = admin.getRegions(tn);
    assertEquals(1, regions.size());
    RegionInfo hri = regions.get(0);
    admin.unassign(hri.getRegionName(), true);
    // We should see this table move out of violation after the report expires.
    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30000, 1000, new Waiter.Predicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            SpaceQuotaSnapshot snapshot = getSnapshotForTable(conn, tn);
            if (snapshot == null) {
                return false;
            }
            return !snapshot.getQuotaStatus().isInViolation();
        }
    });
    // The QuotaObserverChore's memory should also show it not in violation.
    final HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
    QuotaSnapshotStore<TableName> tableStore = master.getQuotaObserverChore().getTableSnapshotStore();
    SpaceQuotaSnapshot snapshot = tableStore.getCurrentState(tn);
    assertFalse("Quota should not be in violation", snapshot.getQuotaStatus().isInViolation());
}

Example 58 with HMaster

use of org.apache.hadoop.hbase.master.HMaster in project hbase by apache.

the class TestSpaceQuotaBasicFunctioning method testDisablePolicyQuotaAndViolate.

@Test
public void testDisablePolicyQuotaAndViolate() throws Exception {
    TableName tableName = helper.createTable();
    helper.setQuotaLimit(tableName, SpaceViolationPolicy.DISABLE, 1L);
    helper.writeData(tableName, SpaceQuotaHelperForTests.ONE_MEGABYTE * 2L);
    TEST_UTIL.getConfiguration().setLong("hbase.master.quotas.region.report.retention.millis", 100);
    HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
    MasterQuotaManager quotaManager = master.getMasterQuotaManager();
    // Make sure the master has report for the table.
    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Waiter.Predicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            Map<RegionInfo, Long> regionSizes = quotaManager.snapshotRegionSizes();
            List<RegionInfo> tableRegions = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tableName);
            return regionSizes.containsKey(tableRegions.get(0));
        }
    });
    // Check if disabled table region report present in the map after retention period expired.
    // It should be present after retention period expired.
    final long regionSizes = quotaManager.snapshotRegionSizes().keySet().stream().filter(k -> k.getTable().equals(tableName)).count();
    Assert.assertTrue(regionSizes > 0);
}

Example 59 with HMaster

use of org.apache.hadoop.hbase.master.HMaster in project hbase by apache.

the class TestMasterQuotasObserver method testObserverAddedByDefault.

@Test
public void testObserverAddedByDefault() throws Exception {
    final HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
    final MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
    Set<String> coprocessorNames = cpHost.getCoprocessors();
    assertTrue("Did not find MasterQuotasObserver in list of CPs: " + coprocessorNames, coprocessorNames.contains(MasterQuotasObserver.class.getSimpleName()));
}

Example 60 with HMaster

use of org.apache.hadoop.hbase.master.HMaster in project hbase by apache.

the class TestWALFiltering method testFlushedSequenceIdsSentToHMaster.

@Test
public void testFlushedSequenceIdsSentToHMaster() throws IOException, InterruptedException, org.apache.hbase.thirdparty.com.google.protobuf.ServiceException, ServiceException {
    SortedMap<byte[], Long> allFlushedSequenceIds = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (int i = 0; i < NUM_RS; ++i) {
        flushAllRegions(i);
    }
    Thread.sleep(10000);
    HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
    for (int i = 0; i < NUM_RS; ++i) {
        for (byte[] regionName : getRegionsByServer(i)) {
            if (allFlushedSequenceIds.containsKey(regionName)) {
                GetLastFlushedSequenceIdRequest req = RequestConverter.buildGetLastFlushedSequenceIdRequest(regionName);
                assertEquals((long) allFlushedSequenceIds.get(regionName), master.getMasterRpcServices().getLastFlushedSequenceId(null, req).getLastFlushedSequenceId());
            }
        }
    }
}