use of co.cask.cdap.data2.util.TableId in project cdap by caskdata.
the class HBaseQueueAdmin method dropAllInNamespace.
@Override
public void dropAllInNamespace(NamespaceId namespaceId) throws Exception {
Set<QueueConstants.QueueType> queueTypes = EnumSet.of(QueueConstants.QueueType.QUEUE, QueueConstants.QueueType.SHARDED_QUEUE);
try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
for (QueueConstants.QueueType queueType : queueTypes) {
// Note: The trailing "." is crucial, since otherwise nsId could match nsId1, nsIdx etc
// It's important to keep config table enabled while disabling and dropping queue tables.
final String queueTableNamePrefix = String.format("%s.%s.", NamespaceId.SYSTEM.getNamespace(), queueType);
final String hbaseNamespace = tableUtil.getHBaseNamespace(namespaceId);
final TableId configTableId = TableId.from(hbaseNamespace, getConfigTableName());
tableUtil.deleteAllInNamespace(ddlExecutor, hbaseNamespace, hConf, new Predicate<TableId>() {
@Override
public boolean apply(TableId tableId) {
// It's a bit hacky here since we know how the Dataset System names tables
return (tableId.getTableName().startsWith(queueTableNamePrefix)) && !tableId.equals(configTableId);
}
});
}
}
// Delete the state store in the namespace
DatasetId id = getStateStoreId(namespaceId.getEntityName());
if (datasetFramework.hasInstance(id)) {
datasetFramework.deleteInstance(id);
}
}
use of co.cask.cdap.data2.util.TableId in project cdap by caskdata.
the class IncrementSummingScannerTest method testIncrementScanning.
@Test
public void testIncrementScanning() throws Exception {
TableId tableId = TableId.from(NamespaceId.DEFAULT.getNamespace(), "TestIncrementSummingScanner");
byte[] familyBytes = Bytes.toBytes("f");
byte[] columnBytes = Bytes.toBytes("c");
HRegion region = createRegion(tableId, familyBytes);
try {
region.initialize();
// test handling of a single increment value alone
Put p = new Put(Bytes.toBytes("r1"));
p.add(familyBytes, columnBytes, Bytes.toBytes(3L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
Scan scan = new Scan();
RegionScanner scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.USER_SCAN);
List<Cell> results = Lists.newArrayList();
scanner.next(results);
assertEquals(1, results.size());
Cell cell = results.get(0);
assertNotNull(cell);
assertEquals(3L, Bytes.toLong(cell.getValue()));
// test handling of a single total sum
p = new Put(Bytes.toBytes("r2"));
p.add(familyBytes, columnBytes, Bytes.toBytes(5L));
region.put(p);
scan = new Scan(Bytes.toBytes("r2"));
scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.USER_SCAN);
results = Lists.newArrayList();
scanner.next(results);
assertEquals(1, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals(5L, Bytes.toLong(cell.getValue()));
// test handling of multiple increment values
long now = System.currentTimeMillis();
p = new Put(Bytes.toBytes("r3"));
for (int i = 0; i < 5; i++) {
p.add(familyBytes, columnBytes, now - i, Bytes.toBytes((long) (i + 1)));
}
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
scan = new Scan(Bytes.toBytes("r3"));
scan.setMaxVersions();
scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.USER_SCAN);
results = Lists.newArrayList();
scanner.next(results);
assertEquals(1, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals(15L, Bytes.toLong(cell.getValue()));
// test handling of multiple increment values followed by a total sum, then other increments
now = System.currentTimeMillis();
p = new Put(Bytes.toBytes("r4"));
for (int i = 0; i < 3; i++) {
p.add(familyBytes, columnBytes, now - i, Bytes.toBytes(1L));
}
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
// this put will appear as a "total" sum prior to all the delta puts
p = new Put(Bytes.toBytes("r4"));
p.add(familyBytes, columnBytes, now - 5, Bytes.toBytes(5L));
region.put(p);
scan = new Scan(Bytes.toBytes("r4"));
scan.setMaxVersions();
scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.USER_SCAN);
results = Lists.newArrayList();
scanner.next(results);
assertEquals(1, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals(8L, Bytes.toLong(cell.getValue()));
// test handling of an increment column followed by a non-increment column
p = new Put(Bytes.toBytes("r4"));
p.add(familyBytes, Bytes.toBytes("c2"), Bytes.toBytes("value"));
region.put(p);
scan = new Scan(Bytes.toBytes("r4"));
scan.setMaxVersions();
scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.USER_SCAN);
results = Lists.newArrayList();
scanner.next(results);
assertEquals(2, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals(8L, Bytes.toLong(cell.getValue()));
cell = results.get(1);
assertNotNull(cell);
assertEquals("value", Bytes.toString(cell.getValue()));
// test handling of an increment column followed by a delete
now = System.currentTimeMillis();
Delete d = new Delete(Bytes.toBytes("r5"));
d.deleteColumn(familyBytes, columnBytes, now - 3);
region.delete(d);
p = new Put(Bytes.toBytes("r5"));
for (int i = 2; i >= 0; i--) {
p.add(familyBytes, columnBytes, now - i, Bytes.toBytes(1L));
}
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
scan = new Scan(Bytes.toBytes("r5"));
scan.setMaxVersions();
scan.setRaw(true);
scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.COMPACT_RETAIN_DELETES);
results = Lists.newArrayList();
scanner.next(results);
// delete marker will not be returned for user scan
assertEquals(2, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals(3L, Bytes.toLong(cell.getValue(), IncrementHandlerState.DELTA_MAGIC_PREFIX.length, 8));
// next cell should be the delete
cell = results.get(1);
assertTrue(CellUtil.isDelete(cell));
} finally {
region.close();
}
}
use of co.cask.cdap.data2.util.TableId in project cdap by caskdata.
the class IncrementSummingScannerTest method createRegion.
static HRegion createRegion(Configuration hConf, CConfiguration cConf, TableId tableId, HColumnDescriptor cfd) throws Exception {
HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get();
HTableDescriptorBuilder htd = tableUtil.buildHTableDescriptor(tableId);
cfd.setMaxVersions(Integer.MAX_VALUE);
cfd.setKeepDeletedCells(true);
htd.addFamily(cfd);
htd.addCoprocessor(IncrementHandler.class.getName());
HTableDescriptor desc = htd.build();
String tableName = desc.getNameAsString();
Path tablePath = new Path("/tmp/" + tableName);
Path hlogPath = new Path("/tmp/hlog-" + tableName);
FileSystem fs = FileSystem.get(hConf);
assertTrue(fs.mkdirs(tablePath));
WALFactory walFactory = new WALFactory(hConf, null, hlogPath.toString());
WAL hLog = walFactory.getWAL(new byte[] { 1 });
HRegionInfo regionInfo = new HRegionInfo(desc.getTableName());
HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(hConf, fs, tablePath, regionInfo);
return new HRegion(regionFS, hLog, hConf, desc, new LocalRegionServerServices(hConf, ServerName.valueOf(InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
use of co.cask.cdap.data2.util.TableId in project cdap by caskdata.
the class IncrementSummingScannerTest method testIncrementScanningWithBatchAndUVB.
@Test
public void testIncrementScanningWithBatchAndUVB() throws Exception {
TableId tableId = TableId.from(NamespaceId.DEFAULT.getNamespace(), "TestIncrementSummingScannerWithUpperVisibilityBound");
byte[] familyBytes = Bytes.toBytes("f");
byte[] columnBytes = Bytes.toBytes("c");
HRegion region = createRegion(tableId, familyBytes);
try {
region.initialize();
long start = 0;
long now = start;
long counter1 = 0;
// adding 5 delta increments
for (int i = 0; i < 5; i++) {
Put p = new Put(Bytes.toBytes("r1"), now++);
p.add(familyBytes, columnBytes, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
counter1++;
}
// Also: we want different combinations of batch limit & uvbs
for (int i = 0; i < 7; i++) {
for (int k = 0; k < 4; k++) {
long[] uvbs = new long[k];
for (int l = 0; l < uvbs.length; l++) {
uvbs[l] = start + (k + 1) * (l + 1);
}
verifyCounts(region, new Scan().setMaxVersions(), new long[] { counter1 }, i > 0 ? i : -1, uvbs);
}
}
// Now test same with two groups of increments
int counter2 = 0;
for (int i = 0; i < 5; i++) {
Put p = new Put(Bytes.toBytes("r2"), now + i);
p.add(familyBytes, columnBytes, Bytes.toBytes(2L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
counter2 += 2;
}
for (int i = 0; i < 12; i++) {
for (int k = 0; k < 4; k++) {
long[] uvbs = new long[k];
for (int l = 0; l < uvbs.length; l++) {
uvbs[l] = start + (k + 1) * (l + 1);
}
verifyCounts(region, new Scan().setMaxVersions(), new long[] { counter1, counter2 }, i > 0 ? i : -1, uvbs);
}
}
} finally {
region.close();
}
}
use of co.cask.cdap.data2.util.TableId in project cdap by caskdata.
the class HBase11TableUtil method listTablesInNamespace.
@Override
public List<TableId> listTablesInNamespace(HBaseAdmin admin, String namespaceId) throws IOException {
List<TableId> tableIds = Lists.newArrayList();
HTableDescriptor[] hTableDescriptors = admin.listTableDescriptorsByNamespace(HTableNameConverter.encodeHBaseEntity(namespaceId));
for (HTableDescriptor hTableDescriptor : hTableDescriptors) {
if (isCDAPTable(hTableDescriptor)) {
tableIds.add(HTableNameConverter.from(hTableDescriptor));
}
}
return tableIds;
}
Aggregations