Examples with TableDescriptor org.apache.hadoop.hbase.client.TableDescriptor used on opensource projects

Example 1 with TableDescriptor

use of org.apache.hadoop.hbase.client.TableDescriptor in project hbase by apache.

the class TestClassLoading method testClassLoadingFromLocalFS.

@Test
public // HBASE-3516: Test CP Class loading from local file system
void testClassLoadingFromLocalFS() throws Exception {
    File jarFile = buildCoprocessorJar(cpName3);
    // create a table that references the jar
    TableDescriptorBuilder tdb = TableDescriptorBuilder.newBuilder(TableName.valueOf(cpName3));
    tdb.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("test")).build());
    tdb.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" + Coprocessor.PRIORITY_USER);
    TableDescriptor tableDescriptor = tdb.build();
    Admin admin = TEST_UTIL.getAdmin();
    admin.createTable(tableDescriptor);
    waitForTable(tableDescriptor.getTableName());
    // verify that the coprocessor was loaded
    boolean found = false;
    SingleProcessHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
    for (HRegion region : hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
        if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName3)) {
            found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
        }
    }
    assertTrue("Class " + cpName3 + " was missing on a region", found);
}

Example 2 with TableDescriptor

use of org.apache.hadoop.hbase.client.TableDescriptor in project hbase by apache.

the class TestClassLoading method loadingClassFromLibDirInJar.

void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
    FileSystem fs = cluster.getFileSystem();
    File innerJarFile1 = buildCoprocessorJar(cpName1);
    File innerJarFile2 = buildCoprocessorJar(cpName2);
    File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
    ClassLoaderTestHelper.addJarFilesToJar(outerJarFile, libPrefix, innerJarFile1, innerJarFile2);
    // copy the jars into dfs
    fs.copyFromLocalFile(new Path(outerJarFile.getPath()), new Path(fs.getUri().toString() + Path.SEPARATOR));
    String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR + outerJarFile.getName();
    assertTrue("Copy jar file to HDFS failed.", fs.exists(new Path(jarFileOnHDFS)));
    LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
    // create a table that references the coprocessors
    TableDescriptorBuilder tdb = TableDescriptorBuilder.newBuilder(tableName);
    tdb.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("test")).build());
    // without configuration values
    tdb.setValue("COPROCESSOR$1", jarFileOnHDFS + "|" + cpName1 + "|" + Coprocessor.PRIORITY_USER);
    // with configuration values
    tdb.setValue("COPROCESSOR$2", jarFileOnHDFS + "|" + cpName2 + "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
    Admin admin = TEST_UTIL.getAdmin();
    if (admin.tableExists(tableName)) {
        if (admin.isTableEnabled(tableName)) {
            admin.disableTable(tableName);
        }
        admin.deleteTable(tableName);
    }
    TableDescriptor tableDescriptor = tdb.build();
    admin.createTable(tableDescriptor);
    waitForTable(tableDescriptor.getTableName());
    // verify that the coprocessors were loaded
    boolean found1 = false, found2 = false, found2_k1 = false, found2_k2 = false, found2_k3 = false;
    SingleProcessHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
    for (HRegion region : hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
        if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
            CoprocessorEnvironment env;
            env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
            if (env != null) {
                found1 = true;
            }
            env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
            if (env != null) {
                found2 = true;
                Configuration conf = env.getConfiguration();
                found2_k1 = conf.get("k1") != null;
                found2_k2 = conf.get("k2") != null;
                found2_k3 = conf.get("k3") != null;
            }
        }
    }
    assertTrue("Class " + cpName1 + " was missing on a region", found1);
    assertTrue("Class " + cpName2 + " was missing on a region", found2);
    assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
    assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
    assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
}

Example 3 with TableDescriptor

use of org.apache.hadoop.hbase.client.TableDescriptor in project hbase by apache.

the class TestCoprocessorEndpoint method setupBeforeClass.

@BeforeClass
public static void setupBeforeClass() throws Exception {
    // set configure to indicate which cp should be loaded
    Configuration conf = util.getConfiguration();
    conf.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 5000);
    conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, org.apache.hadoop.hbase.coprocessor.ColumnAggregationEndpoint.class.getName(), ProtobufCoprocessorService.class.getName());
    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, ProtobufCoprocessorService.class.getName());
    util.startMiniCluster(2);
    Admin admin = util.getAdmin();
    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TEST_TABLE).setColumnFamily(ColumnFamilyDescriptorBuilder.of(TEST_FAMILY)).build();
    admin.createTable(tableDescriptor, new byte[][] { ROWS[rowSeperator1], ROWS[rowSeperator2] });
    util.waitUntilAllRegionsAssigned(TEST_TABLE);
    Table table = util.getConnection().getTable(TEST_TABLE);
    for (int i = 0; i < ROWSIZE; i++) {
        Put put = new Put(ROWS[i]);
        put.addColumn(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(i));
        table.put(put);
    }
    table.close();
}

Example 4 with TableDescriptor

use of org.apache.hadoop.hbase.client.TableDescriptor in project hbase by apache.

the class TestSecureExport method testVisibilityLabels.

@Test
// See HBASE-23990
@org.junit.Ignore
public void testVisibilityLabels() throws IOException, Throwable {
    final String exportTable = name.getMethodName() + "_export";
    final String importTable = name.getMethodName() + "_import";
    final TableDescriptor exportHtd = TableDescriptorBuilder.newBuilder(TableName.valueOf(exportTable)).setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILYA)).build();
    User owner = User.createUserForTesting(UTIL.getConfiguration(), USER_OWNER, new String[0]);
    SecureTestUtil.createTable(UTIL, owner, exportHtd, new byte[][] { Bytes.toBytes("s") });
    AccessTestAction putAction = () -> {
        Put p1 = new Put(ROW1);
        p1.addColumn(FAMILYA, QUAL, NOW, QUAL);
        p1.setCellVisibility(new CellVisibility(SECRET));
        Put p2 = new Put(ROW2);
        p2.addColumn(FAMILYA, QUAL, NOW, QUAL);
        p2.setCellVisibility(new CellVisibility(PRIVATE + " & " + CONFIDENTIAL));
        Put p3 = new Put(ROW3);
        p3.addColumn(FAMILYA, QUAL, NOW, QUAL);
        p3.setCellVisibility(new CellVisibility("!" + CONFIDENTIAL + " & " + TOPSECRET));
        try (Connection conn = ConnectionFactory.createConnection(UTIL.getConfiguration());
            Table t = conn.getTable(TableName.valueOf(exportTable))) {
            t.put(p1);
            t.put(p2);
            t.put(p3);
        }
        return null;
    };
    SecureTestUtil.verifyAllowed(putAction, getUserByLogin(USER_OWNER));
    List<Pair<List<String>, Integer>> labelsAndRowCounts = new LinkedList<>();
    labelsAndRowCounts.add(new Pair<>(Arrays.asList(SECRET), 1));
    labelsAndRowCounts.add(new Pair<>(Arrays.asList(PRIVATE, CONFIDENTIAL), 1));
    labelsAndRowCounts.add(new Pair<>(Arrays.asList(TOPSECRET), 1));
    labelsAndRowCounts.add(new Pair<>(Arrays.asList(TOPSECRET, CONFIDENTIAL), 0));
    labelsAndRowCounts.add(new Pair<>(Arrays.asList(TOPSECRET, CONFIDENTIAL, PRIVATE, SECRET), 2));
    for (final Pair<List<String>, Integer> labelsAndRowCount : labelsAndRowCounts) {
        final List<String> labels = labelsAndRowCount.getFirst();
        final int rowCount = labelsAndRowCount.getSecond();
        // create a open permission directory.
        final Path openDir = new Path("testAccessCase");
        final FileSystem fs = openDir.getFileSystem(UTIL.getConfiguration());
        fs.mkdirs(openDir);
        fs.setPermission(openDir, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
        final Path output = fs.makeQualified(new Path(openDir, "output"));
        AccessTestAction exportAction = () -> {
            StringBuilder buf = new StringBuilder();
            labels.forEach(v -> buf.append(v).append(","));
            buf.deleteCharAt(buf.length() - 1);
            try {
                String[] args = new String[] { "-D " + ExportUtils.EXPORT_VISIBILITY_LABELS + "=" + buf.toString(), exportTable, output.toString() };
                Export.run(new Configuration(UTIL.getConfiguration()), args);
                return null;
            } catch (ServiceException | IOException ex) {
                throw ex;
            } catch (Throwable ex) {
                throw new Exception(ex);
            }
        };
        SecureTestUtil.verifyAllowed(exportAction, getUserByLogin(USER_OWNER));
        final TableDescriptor importHtd = TableDescriptorBuilder.newBuilder(TableName.valueOf(importTable)).setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILYB)).build();
        SecureTestUtil.createTable(UTIL, owner, importHtd, new byte[][] { Bytes.toBytes("s") });
        AccessTestAction importAction = () -> {
            String[] args = new String[] { "-D" + Import.CF_RENAME_PROP + "=" + FAMILYA_STRING + ":" + FAMILYB_STRING, importTable, output.toString() };
            assertEquals(0, ToolRunner.run(new Configuration(UTIL.getConfiguration()), new Import(), args));
            return null;
        };
        SecureTestUtil.verifyAllowed(importAction, getUserByLogin(USER_OWNER));
        AccessTestAction scanAction = () -> {
            Scan scan = new Scan();
            scan.setAuthorizations(new Authorizations(labels));
            try (Connection conn = ConnectionFactory.createConnection(UTIL.getConfiguration());
                Table table = conn.getTable(importHtd.getTableName());
                ResultScanner scanner = table.getScanner(scan)) {
                int count = 0;
                for (Result r : scanner) {
                    ++count;
                }
                assertEquals(rowCount, count);
            }
            return null;
        };
        SecureTestUtil.verifyAllowed(scanAction, getUserByLogin(USER_OWNER));
        AccessTestAction deleteAction = () -> {
            UTIL.deleteTable(importHtd.getTableName());
            return null;
        };
        SecureTestUtil.verifyAllowed(deleteAction, getUserByLogin(USER_OWNER));
        clearOutput(output);
    }
    AccessTestAction deleteAction = () -> {
        UTIL.deleteTable(exportHtd.getTableName());
        return null;
    };
    SecureTestUtil.verifyAllowed(deleteAction, getUserByLogin(USER_OWNER));
}

Example 5 with TableDescriptor

use of org.apache.hadoop.hbase.client.TableDescriptor in project hbase by apache.

the class ExportEndpointExample method main.

public static void main(String[] args) throws Throwable {
    int rowCount = 100;
    byte[] family = Bytes.toBytes("family");
    Configuration conf = HBaseConfiguration.create();
    TableName tableName = TableName.valueOf("ExportEndpointExample");
    try (Connection con = ConnectionFactory.createConnection(conf);
        Admin admin = con.getAdmin()) {
        TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName).setCoprocessor(Export.class.getName()).setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
        admin.createTable(desc);
        List<Put> puts = new ArrayList<>(rowCount);
        for (int row = 0; row != rowCount; ++row) {
            byte[] bs = Bytes.toBytes(row);
            Put put = new Put(bs);
            put.addColumn(family, bs, bs);
            puts.add(put);
        }
        try (Table table = con.getTable(tableName)) {
            table.put(puts);
        }
        Path output = new Path("/tmp/ExportEndpointExample_output");
        Scan scan = new Scan();
        Map<byte[], Export.Response> result = Export.run(conf, tableName, scan, output);
        final long totalOutputRows = result.values().stream().mapToLong(v -> v.getRowCount()).sum();
        final long totalOutputCells = result.values().stream().mapToLong(v -> v.getCellCount()).sum();
        System.out.println("table:" + tableName);
        System.out.println("output:" + output);
        System.out.println("total rows:" + totalOutputRows);
        System.out.println("total cells:" + totalOutputCells);
    }
}