Examples with NamespaceDescriptor org.apache.hadoop.hbase.NamespaceDescriptor used on opensource projects

Example 46 with NamespaceDescriptor

use of org.apache.hadoop.hbase.NamespaceDescriptor in project hbase by apache.

the class TestQuotaObserverChoreWithMiniCluster method testTableQuotaOverridesNamespaceQuota.

@Test
public void testTableQuotaOverridesNamespaceQuota() throws Exception {
    final String namespace = testName.getMethodName();
    final Admin admin = TEST_UTIL.getAdmin();
    // Ensure the namespace exists
    try {
        admin.getNamespaceDescriptor(namespace);
    } catch (NamespaceNotFoundException e) {
        NamespaceDescriptor desc = NamespaceDescriptor.create(namespace).build();
        admin.createNamespace(desc);
    }
    TableName tn1 = helper.createTableWithRegions(namespace, 5);
    TableName tn2 = helper.createTableWithRegions(namespace, 5);
    final long namespaceSizeLimit = 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
    final SpaceViolationPolicy namespaceViolationPolicy = SpaceViolationPolicy.DISABLE;
    QuotaSettings namespaceSettings = QuotaSettingsFactory.limitNamespaceSpace(namespace, namespaceSizeLimit, namespaceViolationPolicy);
    admin.setQuota(namespaceSettings);
    helper.writeData(tn1, 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
    admin.flush(tn1);
    Map<TableName, SpaceQuotaSnapshot> snapshots = snapshotNotifier.copySnapshots();
    for (int i = 0; i < 5; i++) {
        // Check a few times to make sure we don't prematurely move to violation
        assertEquals("Should not see any quota violations after writing 2MB of data: " + snapshots, 0, numSnapshotsInViolation(snapshots));
        try {
            Thread.sleep(DEFAULT_WAIT_MILLIS);
        } catch (InterruptedException e) {
            LOG.debug("Interrupted while sleeping.", e);
        }
        snapshots = snapshotNotifier.copySnapshots();
    }
    helper.writeData(tn2, 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
    admin.flush(tn2);
    snapshots = snapshotNotifier.copySnapshots();
    while (numSnapshotsInViolation(snapshots) < 2) {
        LOG.debug("Saw fewer violations than desired (expected 2): " + snapshots + ". Current reports: " + master.getMasterQuotaManager().snapshotRegionSizes());
        try {
            Thread.sleep(DEFAULT_WAIT_MILLIS);
        } catch (InterruptedException e) {
            LOG.debug("Interrupted while sleeping.", e);
            Thread.currentThread().interrupt();
        }
        snapshots = snapshotNotifier.copySnapshots();
    }
    SpaceQuotaSnapshot actualPolicyTN1 = snapshots.get(tn1);
    assertNotNull("Expected to see violation policy for tn1", actualPolicyTN1);
    assertEquals(namespaceViolationPolicy, actualPolicyTN1.getQuotaStatus().getPolicy().get());
    SpaceQuotaSnapshot actualPolicyTN2 = snapshots.get(tn2);
    assertNotNull("Expected to see violation policy for tn2", actualPolicyTN2);
    assertEquals(namespaceViolationPolicy, actualPolicyTN2.getQuotaStatus().getPolicy().get());
    // Override the namespace quota with a table quota
    final long tableSizeLimit = SpaceQuotaHelperForTests.ONE_MEGABYTE;
    final SpaceViolationPolicy tableViolationPolicy = SpaceViolationPolicy.NO_INSERTS;
    QuotaSettings tableSettings = QuotaSettingsFactory.limitTableSpace(tn1, tableSizeLimit, tableViolationPolicy);
    admin.setQuota(tableSettings);
    // Keep checking for the table quota policy to override the namespace quota
    while (true) {
        snapshots = snapshotNotifier.copySnapshots();
        SpaceQuotaSnapshot actualTableSnapshot = snapshots.get(tn1);
        assertNotNull("Violation policy should never be null", actualTableSnapshot);
        if (tableViolationPolicy != actualTableSnapshot.getQuotaStatus().getPolicy().orElse(null)) {
            LOG.debug("Saw unexpected table violation policy, waiting and re-checking.");
            try {
                Thread.sleep(DEFAULT_WAIT_MILLIS);
            } catch (InterruptedException e) {
                LOG.debug("Interrupted while sleeping");
                Thread.currentThread().interrupt();
            }
            continue;
        }
        assertEquals(tableViolationPolicy, actualTableSnapshot.getQuotaStatus().getPolicy().get());
        break;
    }
    // This should not change with the introduction of the table quota for tn1
    actualPolicyTN2 = snapshots.get(tn2);
    assertNotNull("Expected to see violation policy for tn2", actualPolicyTN2);
    assertEquals(namespaceViolationPolicy, actualPolicyTN2.getQuotaStatus().getPolicy().get());
}

Example 47 with NamespaceDescriptor

use of org.apache.hadoop.hbase.NamespaceDescriptor in project ranger by apache.

the class HBaseRangerAuthorizationTest method setup.

@org.junit.BeforeClass
public static void setup() throws Exception {
    port = getFreePort();
    utility = new HBaseTestingUtility();
    utility.getConfiguration().set("test.hbase.zookeeper.property.clientPort", "" + port);
    utility.getConfiguration().set("hbase.master.port", "" + getFreePort());
    utility.getConfiguration().set("hbase.master.info.port", "" + getFreePort());
    utility.getConfiguration().set("hbase.regionserver.port", "" + getFreePort());
    utility.getConfiguration().set("hbase.regionserver.info.port", "" + getFreePort());
    utility.getConfiguration().set("zookeeper.znode.parent", "/hbase-unsecure");
    // Enable authorization
    utility.getConfiguration().set("hbase.security.authorization", "true");
    utility.getConfiguration().set("hbase.coprocessor.master.classes", "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor");
    utility.getConfiguration().set("hbase.coprocessor.region.classes", "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor");
    utility.startMiniCluster();
    // Create a table as "admin"
    final Configuration conf = HBaseConfiguration.create();
    conf.set("hbase.zookeeper.quorum", "localhost");
    conf.set("hbase.zookeeper.property.clientPort", "" + port);
    conf.set("zookeeper.znode.parent", "/hbase-unsecure");
    // Create a table
    Connection conn = ConnectionFactory.createConnection(conf);
    Admin admin = conn.getAdmin();
    // Create a table
    if (!admin.tableExists(TableName.valueOf("default:temp"))) {
        HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf("default:temp"));
        // Adding column families to table descriptor
        tableDescriptor.addFamily(new HColumnDescriptor("colfam1"));
        tableDescriptor.addFamily(new HColumnDescriptor("colfam2"));
        admin.createTable(tableDescriptor);
    }
    if (!admin.tableExists(TableName.valueOf("default:temp5"))) {
        HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf("default:temp5"));
        // Adding column families to table descriptor
        tableDescriptor.addFamily(new HColumnDescriptor("colfam1"));
        admin.createTable(tableDescriptor);
    }
    // Add a new row
    Put put = new Put(Bytes.toBytes("row1"));
    put.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("col1"), Bytes.toBytes("val1"));
    Table table = conn.getTable(TableName.valueOf("temp"));
    table.put(put);
    put = new Put(Bytes.toBytes("row1"));
    put.addColumn(Bytes.toBytes("colfam2"), Bytes.toBytes("col1"), Bytes.toBytes("val2"));
    table.put(put);
    // Create a namespace
    NamespaceDescriptor ns = NamespaceDescriptor.create("test_namespace").build();
    admin.createNamespace(ns);
    // Create a table
    if (!admin.tableExists(TableName.valueOf("test_namespace", "temp"))) {
        HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf("test_namespace", "temp"));
        // Adding column families to table descriptor
        tableDescriptor.addFamily(new HColumnDescriptor("colfam1"));
        tableDescriptor.addFamily(new HColumnDescriptor("colfam2"));
        admin.createTable(tableDescriptor);
    }
    table = conn.getTable(TableName.valueOf("test_namespace", "temp"));
    // Add a new row
    put = new Put(Bytes.toBytes("row1"));
    put.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("col1"), Bytes.toBytes("val1"));
    table.put(put);
    put = new Put(Bytes.toBytes("row1"));
    put.addColumn(Bytes.toBytes("colfam2"), Bytes.toBytes("col1"), Bytes.toBytes("val2"));
    table.put(put);
    conn.close();
}

Example 48 with NamespaceDescriptor

use of org.apache.hadoop.hbase.NamespaceDescriptor in project hbase by apache.

the class TableNamespaceManager method isTableAvailableAndInitialized.

/**
   * This method checks if the namespace table is assigned and then
   * tries to create its Table reference. If it was already created before, it also makes
   * sure that the connection isn't closed.
   * @return true if the namespace table manager is ready to serve, false otherwise
   */
@SuppressWarnings("deprecation")
public synchronized boolean isTableAvailableAndInitialized() throws IOException {
    // Did we already get a table? If so, still make sure it's available
    if (isTableNamespaceManagerInitialized()) {
        return true;
    }
    // Now check if the table is assigned, if not then fail fast
    if (isTableAssigned() && isTableEnabled()) {
        try {
            boolean initGoodSofar = true;
            nsTable = this.masterServices.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME);
            zkNamespaceManager = new ZKNamespaceManager(masterServices.getZooKeeper());
            zkNamespaceManager.start();
            if (get(nsTable, NamespaceDescriptor.DEFAULT_NAMESPACE.getName()) == null) {
                blockingCreateNamespace(NamespaceDescriptor.DEFAULT_NAMESPACE);
            }
            if (get(nsTable, NamespaceDescriptor.SYSTEM_NAMESPACE.getName()) == null) {
                blockingCreateNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE);
            }
            if (!initGoodSofar) {
                // some required namespace is created asynchronized. We should complete init later.
                return false;
            }
            ResultScanner scanner = nsTable.getScanner(HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES);
            try {
                for (Result result : scanner) {
                    byte[] val = CellUtil.cloneValue(result.getColumnLatestCell(HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES, HTableDescriptor.NAMESPACE_COL_DESC_BYTES));
                    NamespaceDescriptor ns = ProtobufUtil.toNamespaceDescriptor(HBaseProtos.NamespaceDescriptor.parseFrom(val));
                    zkNamespaceManager.update(ns);
                }
            } finally {
                scanner.close();
            }
            initialized = true;
            return true;
        } catch (IOException ie) {
            LOG.warn("Caught exception in initializing namespace table manager", ie);
            if (nsTable != null) {
                nsTable.close();
            }
            throw ie;
        }
    }
    return false;
}

Example 49 with NamespaceDescriptor

use of org.apache.hadoop.hbase.NamespaceDescriptor in project hbase by apache.

the class TestNamespaceAuditor method testRegionOperations.

@Test
public void testRegionOperations() throws Exception {
    String nsp1 = prefix + "_regiontest";
    NamespaceDescriptor nspDesc = NamespaceDescriptor.create(nsp1).addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2").addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
    ADMIN.createNamespace(nspDesc);
    boolean constraintViolated = false;
    final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
    byte[] columnFamily = Bytes.toBytes("info");
    HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
    tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
    NamespaceTableAndRegionInfo stateInfo;
    try {
        ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 7);
    } catch (Exception exp) {
        assertTrue(exp instanceof DoNotRetryIOException);
        LOG.info(exp);
        constraintViolated = true;
    } finally {
        assertTrue(constraintViolated);
    }
    assertFalse(ADMIN.tableExists(tableOne));
    // This call will pass.
    ADMIN.createTable(tableDescOne);
    Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
    Table htable = connection.getTable(tableOne);
    UTIL.loadNumericRows(htable, Bytes.toBytes("info"), 1, 1000);
    ADMIN.flush(tableOne);
    stateInfo = getNamespaceState(nsp1);
    assertEquals(1, stateInfo.getTables().size());
    assertEquals(1, stateInfo.getRegionCount());
    restartMaster();
    HRegion actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
    CustomObserver observer = (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(CustomObserver.class.getName());
    assertNotNull(observer);
    ADMIN.split(tableOne, Bytes.toBytes("500"));
    observer.postSplit.await();
    assertEquals(2, ADMIN.getTableRegions(tableOne).size());
    actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
    observer = (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(CustomObserver.class.getName());
    assertNotNull(observer);
    //Before we go on split, we should remove all reference store files.
    ADMIN.compact(tableOne);
    observer.postCompact.await();
    ADMIN.split(tableOne, getSplitKey(actualRegion.getRegionInfo().getStartKey(), actualRegion.getRegionInfo().getEndKey()));
    observer.postSplit.await();
    // Make sure no regions have been added.
    List<HRegionInfo> hris = ADMIN.getTableRegions(tableOne);
    assertEquals(2, hris.size());
    htable.close();
}

Example 50 with NamespaceDescriptor

use of org.apache.hadoop.hbase.NamespaceDescriptor in project atlas by apache.

the class HBaseBridge method importTable.

public void importTable(final String tableName) throws Exception {
    String tableNameStr = null;
    HTableDescriptor[] htds = hbaseAdmin.listTables(Pattern.compile(tableName));
    if (ArrayUtils.isNotEmpty(htds)) {
        for (HTableDescriptor htd : htds) {
            String tblNameWithNameSpace = htd.getTableName().getNameWithNamespaceInclAsString();
            String tblNameWithOutNameSpace = htd.getTableName().getNameAsString();
            if (tableName.equals(tblNameWithNameSpace)) {
                tableNameStr = tblNameWithNameSpace;
            } else if (tableName.equals(tblNameWithOutNameSpace)) {
                tableNameStr = tblNameWithOutNameSpace;
            } else {
                // when wild cards are used in table name
                if (tblNameWithNameSpace != null) {
                    tableNameStr = tblNameWithNameSpace;
                } else if (tblNameWithOutNameSpace != null) {
                    tableNameStr = tblNameWithOutNameSpace;
                }
            }
            byte[] nsByte = htd.getTableName().getNamespace();
            String nsName = new String(nsByte);
            NamespaceDescriptor nsDescriptor = hbaseAdmin.getNamespaceDescriptor(nsName);
            AtlasEntityWithExtInfo entity = createOrUpdateNameSpace(nsDescriptor);
            HColumnDescriptor[] hcdts = htd.getColumnFamilies();
            createOrUpdateTable(nsName, tableNameStr, entity.getEntity(), htd, hcdts);
        }
    } else {
        throw new AtlasHookException("No Table found for the given criteria. Table = " + tableName);
    }
}