use of io.cdap.cdap.data2.util.hbase.HTableDescriptorBuilder in project cdap by caskdata.
the class IncrementHandlerTest method createTable.
@Override
public HTable createTable(TableId tableId) throws Exception {
HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get();
HTableDescriptorBuilder tableDesc = tableUtil.buildHTableDescriptor(tableId);
HColumnDescriptor columnDesc = new HColumnDescriptor(FAMILY);
columnDesc.setMaxVersions(Integer.MAX_VALUE);
columnDesc.setValue(IncrementHandlerState.PROPERTY_TRANSACTIONAL, "false");
tableDesc.addFamily(columnDesc);
tableDesc.addCoprocessor(IncrementHandler.class.getName());
HTableDescriptor htd = tableDesc.build();
TEST_HBASE.getHBaseAdmin().createTable(htd);
TEST_HBASE.waitUntilTableAvailable(htd.getName(), 5000);
return tableUtil.createHTable(conf, tableId);
}
use of io.cdap.cdap.data2.util.hbase.HTableDescriptorBuilder in project cdap by caskdata.
the class IncrementHandlerTest method createTable.
@Override
public Table createTable(TableId tableId) throws Exception {
HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get();
HTableDescriptorBuilder tableDesc = tableUtil.buildHTableDescriptor(tableId);
HColumnDescriptor columnDesc = new HColumnDescriptor(FAMILY);
columnDesc.setMaxVersions(Integer.MAX_VALUE);
columnDesc.setValue(IncrementHandlerState.PROPERTY_TRANSACTIONAL, "false");
tableDesc.addFamily(columnDesc);
tableDesc.addCoprocessor(IncrementHandler.class.getName());
HTableDescriptor htd = tableDesc.build();
TEST_HBASE.getHBaseAdmin().createTable(htd);
TEST_HBASE.waitUntilTableAvailable(htd.getName(), 5000);
return tableUtil.createTable(conf, tableId);
}
use of io.cdap.cdap.data2.util.hbase.HTableDescriptorBuilder in project cdap by caskdata.
the class AbstractHBaseDataSetAdmin method updateTable.
/**
* Performs update on a given HBase table. It will be updated if either its spec has
* changed since the HBase table was created or updated, or if the CDAP version recorded
* in the HTable descriptor is less than the current CDAP version.
*
* @param force forces update regardless of whether the table needs it.
* @throws IOException If update failed.
*/
public void updateTable(boolean force) throws IOException {
try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
HTableDescriptor tableDescriptor;
HTableDescriptorBuilder newDescriptor;
try (HBaseAdmin admin = new HBaseAdmin(hConf)) {
tableDescriptor = tableUtil.getHTableDescriptor(admin, tableId);
// create a new descriptor for the table update
newDescriptor = tableUtil.buildHTableDescriptor(tableDescriptor);
}
// update any table properties if necessary
boolean needUpdate = needsUpdate(tableDescriptor, newDescriptor) || force;
// Get the cdap version from the table
ProjectInfo.Version version = HBaseTableUtil.getVersion(tableDescriptor);
String hbaseVersion = HBaseTableUtil.getHBaseVersion(tableDescriptor);
if (!needUpdate && hbaseVersion != null && hbaseVersion.equals(HBaseVersion.getVersionString()) && version.compareTo(ProjectInfo.getVersion()) >= 0) {
// If neither the table spec nor the cdap version have changed, no need to update
LOG.info("Table '{}' has not changed and its version '{}' is same or greater " + "than current CDAP version '{}'. The underlying HBase version {} has also not changed.", tableId, version, ProjectInfo.getVersion(), hbaseVersion);
return;
}
// Generate the coprocessor jar
CoprocessorJar coprocessorJar = createCoprocessorJar();
Location jarLocation = coprocessorJar.getJarLocation();
// Check if coprocessor upgrade is needed
Map<String, HBaseTableUtil.CoprocessorInfo> coprocessorInfo = HBaseTableUtil.getCoprocessorInfo(tableDescriptor);
// For all required coprocessors, check if they've need to be upgraded.
for (Class<? extends Coprocessor> coprocessor : coprocessorJar.getCoprocessors()) {
HBaseTableUtil.CoprocessorInfo info = coprocessorInfo.get(coprocessor.getName());
if (info != null) {
// The same coprocessor has been configured, check by the file name to see if they are the same.
if (!jarLocation.getName().equals(info.getPath().getName())) {
// Remove old one and add the new one.
newDescriptor.removeCoprocessor(info.getClassName());
addCoprocessor(newDescriptor, coprocessor, coprocessorJar.getPriority(coprocessor));
}
} else {
// The coprocessor is missing from the table, add it.
addCoprocessor(newDescriptor, coprocessor, coprocessorJar.getPriority(coprocessor));
}
}
// Removes all old coprocessors
Set<String> coprocessorNames = ImmutableSet.copyOf(Iterables.transform(coprocessorJar.coprocessors, CLASS_TO_NAME));
for (String remove : Sets.difference(coprocessorInfo.keySet(), coprocessorNames)) {
newDescriptor.removeCoprocessor(remove);
}
HBaseTableUtil.setVersion(newDescriptor);
HBaseTableUtil.setHBaseVersion(newDescriptor);
HBaseTableUtil.setTablePrefix(newDescriptor, cConf);
LOG.info("Updating table '{}'...", tableId);
TableName tableName = HTableNameConverter.toTableName(cConf.get(Constants.Dataset.TABLE_PREFIX), tableId);
boolean enableTable = false;
try {
ddlExecutor.disableTableIfEnabled(tableName.getNamespaceAsString(), tableName.getQualifierAsString());
enableTable = true;
} catch (TableNotEnabledException e) {
// If the table is in cdap_system namespace enable it regardless so that they can be used later. See CDAP-7324
if (isSystemTable()) {
enableTable = true;
} else {
LOG.debug("Table '{}' was not enabled before update and will not be enabled after update.", tableId);
}
}
tableUtil.modifyTable(ddlExecutor, newDescriptor.build());
if (enableTable) {
LOG.debug("Enabling table '{}'...", tableId);
ddlExecutor.enableTableIfDisabled(tableName.getNamespaceAsString(), tableName.getQualifierAsString());
}
}
LOG.info("Table '{}' update completed.", tableId);
}
use of io.cdap.cdap.data2.util.hbase.HTableDescriptorBuilder in project cdap by caskdata.
the class IncrementHandlerTest method createTable.
@Override
public Table createTable(TableId tableId) throws Exception {
HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get();
HTableDescriptorBuilder tableDesc = tableUtil.buildHTableDescriptor(tableId);
HColumnDescriptor columnDesc = new HColumnDescriptor(FAMILY);
columnDesc.setMaxVersions(Integer.MAX_VALUE);
columnDesc.setValue(IncrementHandlerState.PROPERTY_TRANSACTIONAL, "false");
tableDesc.addFamily(columnDesc);
tableDesc.addCoprocessor(IncrementHandler.class.getName());
HTableDescriptor htd = tableDesc.build();
TEST_HBASE.getHBaseAdmin().createTable(htd);
TEST_HBASE.waitUntilTableAvailable(htd.getName(), 5000);
return tableUtil.createTable(conf, tableId);
}
use of io.cdap.cdap.data2.util.hbase.HTableDescriptorBuilder in project cdap by caskdata.
the class AbstractHBaseTableUtilTest method testTableSizeMetrics.
@Test
public void testTableSizeMetrics() throws Exception {
HBaseTableUtil tableUtil = getTableUtil();
// namespace should not exist
if (namespacesSupported()) {
Assert.assertFalse(tableUtil.hasNamespace(hAdmin, tableUtil.getHBaseNamespace(new NamespaceId("namespace"))));
}
Assert.assertNull(getTableStats("namespace", "table1"));
Assert.assertNull(getTableStats("namespace", "table2"));
Assert.assertNull(getTableStats("namespace", "table3"));
if (namespacesSupported()) {
createNamespace("namespace");
createNamespace("namespace2");
Assert.assertTrue(tableUtil.hasNamespace(hAdmin, tableUtil.getHBaseNamespace(new NamespaceId("namespace"))));
}
Futures.allAsList(createAsync(TableId.from("namespace", "table1")), createAsync(TableId.from("namespace2", "table1")), createAsync(TableId.from("namespace", "table2")), createAsync(TableId.from("namespace", "table3"))).get(60, TimeUnit.SECONDS);
Assert.assertTrue(exists("namespace", "table1"));
Assert.assertTrue(exists("namespace2", "table1"));
Assert.assertTrue(exists("namespace", "table2"));
Assert.assertTrue(exists("namespace", "table3"));
Tasks.waitFor(true, new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
try {
Assert.assertEquals(0, getTableStats("namespace", "table1").getTotalSizeMB());
Assert.assertEquals(0, getTableStats("namespace2", "table1").getTotalSizeMB());
Assert.assertEquals(0, getTableStats("namespace", "table2").getTotalSizeMB());
Assert.assertEquals(0, getTableStats("namespace", "table3").getTotalSizeMB());
return true;
} catch (Throwable t) {
return false;
}
}
}, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
writeSome("namespace2", "table1");
writeSome("namespace", "table2");
writeSome("namespace", "table3");
Tasks.waitFor(true, new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
try {
Assert.assertEquals(0, getTableStats("namespace", "table1").getTotalSizeMB());
Assert.assertTrue(getTableStats("namespace2", "table1").getTotalSizeMB() > 0);
Assert.assertTrue(getTableStats("namespace", "table2").getTotalSizeMB() > 0);
Assert.assertTrue(getTableStats("namespace", "table3").getTotalSizeMB() > 0);
return true;
} catch (Throwable t) {
return false;
}
}
}, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
drop("namespace", "table1");
Assert.assertFalse(exists("namespace", "table1"));
TableId hTableId = tableUtil.createHTableId(new NamespaceId("namespace"), "table2");
// TODO: TestHBase methods should eventually accept namespace as a param, but will add them incrementally
TEST_HBASE.forceRegionFlush(Bytes.toBytes(getTableNameAsString(hTableId)));
truncate("namespace", "table3");
Tasks.waitFor(true, new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
try {
Assert.assertNull(getTableStats("namespace", "table1"));
Assert.assertTrue(getTableStats("namespace", "table2").getTotalSizeMB() > 0);
Assert.assertTrue(getTableStats("namespace", "table2").getStoreFileSizeMB() > 0);
Assert.assertEquals(0, getTableStats("namespace", "table3").getTotalSizeMB());
return true;
} catch (Throwable t) {
return false;
}
}
}, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// modify
HTableDescriptor desc = getTableDescriptor("namespace2", "table1");
HTableDescriptorBuilder newDesc = getTableUtil().buildHTableDescriptor(desc);
newDesc.setValue("mykey", "myvalue");
disable("namespace2", "table1");
getTableUtil().modifyTable(ddlExecutor, newDesc.build());
desc = getTableDescriptor("namespace2", "table1");
Assert.assertTrue(desc.getValue("mykey").equals("myvalue"));
enable("namespace2", "table1");
desc = getTableDescriptor("namespace", "table2");
Assert.assertNull(desc.getValue("myKey"));
// Make sure that HBase version is added
Assert.assertNotNull(desc.getValue(HBaseTableUtil.CDAP_HBASE_VERSION));
if (namespacesSupported()) {
try {
deleteNamespace("namespace");
Assert.fail("Should not be able to delete a non-empty namespace.");
} catch (ConstraintException e) {
// Expected exception
}
}
Futures.allAsList(dropAsync(TableId.from("namespace2", "table1")), dropAsync(TableId.from("namespace", "table2")), dropAsync(TableId.from("namespace", "table3"))).get(60, TimeUnit.SECONDS);
if (namespacesSupported()) {
deleteNamespace("namespace");
deleteNamespace("namespace2");
Assert.assertFalse(tableUtil.hasNamespace(hAdmin, "namespace"));
Assert.assertFalse(tableUtil.hasNamespace(hAdmin, "namespace2"));
}
}
Aggregations