Examples with SpaceQuotaStatus org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus used on opensource projects

Example 1 with SpaceQuotaStatus

use of org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus in project hbase by apache.

the class QuotaObserverChore method _chore.

void _chore() throws IOException {
    // Get the total set of tables that have quotas defined. Includes table quotas
    // and tables included by namespace quotas.
    TablesWithQuotas tablesWithQuotas = fetchAllTablesWithQuotasDefined();
    if (LOG.isTraceEnabled()) {
        LOG.trace("Found following tables with quotas: " + tablesWithQuotas);
    }
    if (metrics != null) {
        // Set the number of namespaces and tables with quotas defined
        metrics.setNumSpaceQuotas(tablesWithQuotas.getTableQuotaTables().size() + tablesWithQuotas.getNamespacesWithQuotas().size());
    }
    // The current "view" of region space use. Used henceforth.
    final Map<RegionInfo, Long> reportedRegionSpaceUse = quotaManager.snapshotRegionSizes();
    if (LOG.isTraceEnabled()) {
        LOG.trace("Using " + reportedRegionSpaceUse.size() + " region space use reports: " + reportedRegionSpaceUse);
    }
    // Remove the "old" region reports
    pruneOldRegionReports();
    // Create the stores to track table and namespace snapshots
    initializeSnapshotStores(reportedRegionSpaceUse);
    // Report the number of (non-expired) region size reports
    if (metrics != null) {
        metrics.setNumRegionSizeReports(reportedRegionSpaceUse.size());
    }
    // Filter out tables for which we don't have adequate regionspace reports yet.
    // Important that we do this after we instantiate the stores above
    // This gives us a set of Tables which may or may not be violating their quota.
    // To be safe, we want to make sure that these are not in violation.
    Set<TableName> tablesInLimbo = tablesWithQuotas.filterInsufficientlyReportedTables(tableSnapshotStore);
    if (LOG.isTraceEnabled()) {
        LOG.trace("Filtered insufficiently reported tables, left with " + reportedRegionSpaceUse.size() + " regions reported");
    }
    for (TableName tableInLimbo : tablesInLimbo) {
        final SpaceQuotaSnapshot currentSnapshot = tableSnapshotStore.getCurrentState(tableInLimbo);
        SpaceQuotaStatus currentStatus = currentSnapshot.getQuotaStatus();
        if (currentStatus.isInViolation()) {
            if (LOG.isTraceEnabled()) {
                LOG.trace("Moving " + tableInLimbo + " out of violation because fewer region sizes were" + " reported than required.");
            }
            SpaceQuotaSnapshot targetSnapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), currentSnapshot.getUsage(), currentSnapshot.getLimit());
            this.snapshotNotifier.transitionTable(tableInLimbo, targetSnapshot);
            // Update it in the Table QuotaStore so that memory is consistent with no violation.
            tableSnapshotStore.setCurrentState(tableInLimbo, targetSnapshot);
            // In case of Disable SVP, we need to enable the table as it moves out of violation
            if (SpaceViolationPolicy.DISABLE == currentStatus.getPolicy().orElse(null)) {
                QuotaUtil.enableTableIfNotEnabled(conn, tableInLimbo);
            }
        }
    }
    // Transition each table to/from quota violation based on the current and target state.
    // Only table quotas are enacted.
    final Set<TableName> tablesWithTableQuotas = tablesWithQuotas.getTableQuotaTables();
    processTablesWithQuotas(tablesWithTableQuotas);
    // For each Namespace quota, transition each table in the namespace in or out of violation
    // only if a table quota violation policy has not already been applied.
    final Set<String> namespacesWithQuotas = tablesWithQuotas.getNamespacesWithQuotas();
    final Multimap<String, TableName> tablesByNamespace = tablesWithQuotas.getTablesByNamespace();
    processNamespacesWithQuotas(namespacesWithQuotas, tablesByNamespace);
}

Example 2 with SpaceQuotaStatus

use of org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus in project hbase by apache.

the class SpaceViolationPolicyEnforcementFactory method createWithoutViolation.

/**
 * Creates the "default" {@link SpaceViolationPolicyEnforcement} for a table that isn't in
 * violation. This is used to have uniform policy checking for tables in and not quotas. This
 * policy will still verify that new bulk loads do not exceed the configured quota limit.
 *
 * @param rss RegionServerServices instance the policy enforcement should use.
 * @param tableName The target HBase table.
 * @param snapshot The current quota snapshot for the {@code tableName}, can be null.
 */
public SpaceViolationPolicyEnforcement createWithoutViolation(RegionServerServices rss, TableName tableName, SpaceQuotaSnapshot snapshot) {
    if (snapshot == null) {
        // We should do use the (singleton instance) of this policy to do nothing.
        return MissingSnapshotViolationPolicyEnforcement.getInstance();
    }
    // We have a snapshot which means that there is a quota set on this table, but it's not in
    // violation of that quota. We need to construct a policy for this table.
    SpaceQuotaStatus status = snapshot.getQuotaStatus();
    if (status.isInViolation()) {
        throw new IllegalArgumentException(tableName + " is in violation. Logic error. Snapshot=" + snapshot);
    }
    // We have a unique size snapshot to use. Create an instance for this tablename + snapshot.
    DefaultViolationPolicyEnforcement enforcement = new DefaultViolationPolicyEnforcement();
    enforcement.initialize(rss, tableName, snapshot);
    return enforcement;
}

Example 3 with SpaceQuotaStatus

use of org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus in project hbase by apache.

the class TestMasterMetricsWrapper method testQuotaSnapshotConversion.

@Test
public void testQuotaSnapshotConversion() {
    MetricsMasterWrapperImpl info = new MetricsMasterWrapperImpl(TEST_UTIL.getHBaseCluster().getMaster());
    assertEquals(new SimpleImmutableEntry<Long, Long>(1024L, 2048L), info.convertSnapshot(new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 1024L, 2048L)));
    assertEquals(new SimpleImmutableEntry<Long, Long>(4096L, 2048L), info.convertSnapshot(new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 4096L, 2048L)));
}

Example 4 with SpaceQuotaStatus

use of org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus in project hbase by apache.

the class TestRegionServerSpaceQuotaManager method testSpacePoliciesFromEnforcements.

@Test
public void testSpacePoliciesFromEnforcements() {
    final Map<TableName, SpaceViolationPolicyEnforcement> enforcements = new HashMap<>();
    final Map<TableName, SpaceQuotaSnapshot> expectedPolicies = new HashMap<>();
    when(quotaManager.copyActiveEnforcements()).thenReturn(enforcements);
    when(quotaManager.getActivePoliciesAsMap()).thenCallRealMethod();
    NoInsertsViolationPolicyEnforcement noInsertsPolicy = new NoInsertsViolationPolicyEnforcement();
    SpaceQuotaSnapshot noInsertsSnapshot = new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 256L, 1024L);
    noInsertsPolicy.initialize(rss, TableName.valueOf("no_inserts"), noInsertsSnapshot);
    enforcements.put(noInsertsPolicy.getTableName(), noInsertsPolicy);
    expectedPolicies.put(noInsertsPolicy.getTableName(), noInsertsSnapshot);
    NoWritesViolationPolicyEnforcement noWritesPolicy = new NoWritesViolationPolicyEnforcement();
    SpaceQuotaSnapshot noWritesSnapshot = new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 512L, 2048L);
    noWritesPolicy.initialize(rss, TableName.valueOf("no_writes"), noWritesSnapshot);
    enforcements.put(noWritesPolicy.getTableName(), noWritesPolicy);
    expectedPolicies.put(noWritesPolicy.getTableName(), noWritesSnapshot);
    NoWritesCompactionsViolationPolicyEnforcement noWritesCompactionsPolicy = new NoWritesCompactionsViolationPolicyEnforcement();
    SpaceQuotaSnapshot noWritesCompactionsSnapshot = new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES_COMPACTIONS), 1024L, 4096L);
    noWritesCompactionsPolicy.initialize(rss, TableName.valueOf("no_writes_compactions"), noWritesCompactionsSnapshot);
    enforcements.put(noWritesCompactionsPolicy.getTableName(), noWritesCompactionsPolicy);
    expectedPolicies.put(noWritesCompactionsPolicy.getTableName(), noWritesCompactionsSnapshot);
    DisableTableViolationPolicyEnforcement disablePolicy = new DisableTableViolationPolicyEnforcement();
    SpaceQuotaSnapshot disableSnapshot = new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 2048L, 8192L);
    disablePolicy.initialize(rss, TableName.valueOf("disable"), disableSnapshot);
    enforcements.put(disablePolicy.getTableName(), disablePolicy);
    expectedPolicies.put(disablePolicy.getTableName(), disableSnapshot);
    enforcements.put(TableName.valueOf("no_policy"), new DefaultViolationPolicyEnforcement());
    Map<TableName, SpaceQuotaSnapshot> actualPolicies = quotaManager.getActivePoliciesAsMap();
    assertEquals(expectedPolicies, actualPolicies);
}

Example 5 with SpaceQuotaStatus

use of org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus in project hbase by apache.

the class TestRegionServerSpaceQuotaManager method testExceptionOnPolicyEnforcementDisable.

@Test
public void testExceptionOnPolicyEnforcementDisable() throws Exception {
    final TableName tableName = TableName.valueOf("foo");
    final SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 2048L);
    RegionServerServices rss = mock(RegionServerServices.class);
    SpaceViolationPolicyEnforcementFactory factory = mock(SpaceViolationPolicyEnforcementFactory.class);
    SpaceViolationPolicyEnforcement enforcement = mock(SpaceViolationPolicyEnforcement.class);
    RegionServerSpaceQuotaManager realManager = new RegionServerSpaceQuotaManager(rss, factory);
    when(factory.create(rss, tableName, snapshot)).thenReturn(enforcement);
    doNothing().when(enforcement).enable();
    doThrow(new IOException("Failed for test!")).when(enforcement).disable();
    // Enabling should work
    realManager.enforceViolationPolicy(tableName, snapshot);
    Map<TableName, SpaceViolationPolicyEnforcement> enforcements = realManager.copyActiveEnforcements();
    assertEquals(1, enforcements.size());
    // If the disable fails, we should still treat it as "active"
    realManager.disableViolationPolicyEnforcement(tableName);
    enforcements = realManager.copyActiveEnforcements();
    assertEquals(1, enforcements.size());
}