use of co.cask.cdap.data2.util.TableId in project cdap by caskdata.
the class HBase11TableUtil method listTables.
@Override
public List<TableId> listTables(HBaseAdmin admin) throws IOException {
List<TableId> tableIds = Lists.newArrayList();
HTableDescriptor[] hTableDescriptors = admin.listTables();
for (HTableDescriptor hTableDescriptor : hTableDescriptors) {
if (isCDAPTable(hTableDescriptor)) {
tableIds.add(HTableNameConverter.from(hTableDescriptor));
}
}
return tableIds;
}
use of co.cask.cdap.data2.util.TableId in project cdap by caskdata.
the class IncrementSummingScannerTest method testFlushAndCompact.
@Test
public void testFlushAndCompact() throws Exception {
TableId tableId = TableId.from(NamespaceId.DEFAULT.getNamespace(), "TestFlushAndCompact");
byte[] familyBytes = Bytes.toBytes("f");
byte[] columnBytes = Bytes.toBytes("c");
HRegion region = createRegion(tableId, familyBytes);
try {
region.initialize();
// load an initial set of increments
long ts = System.currentTimeMillis();
byte[] row1 = Bytes.toBytes("row1");
for (int i = 0; i < 50; i++) {
Put p = new Put(row1);
p.add(familyBytes, columnBytes, ts, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
ts++;
region.put(p);
}
byte[] row2 = Bytes.toBytes("row2");
ts = System.currentTimeMillis();
// start with a full put
Put row2P = new Put(row2);
row2P.add(familyBytes, columnBytes, ts++, Bytes.toBytes(10L));
region.put(row2P);
for (int i = 0; i < 10; i++) {
Put p = new Put(row2);
p.add(familyBytes, columnBytes, ts++, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
}
// force a region flush
region.flushcache();
region.waitForFlushesAndCompactions();
Result r1 = region.get(new Get(row1));
assertNotNull(r1);
assertFalse(r1.isEmpty());
// row1 should have a full put aggregating all 50 incrments
Cell r1Cell = r1.getColumnLatestCell(familyBytes, columnBytes);
assertNotNull(r1Cell);
assertEquals(50L, Bytes.toLong(r1Cell.getValue()));
Result r2 = region.get(new Get(row2));
assertNotNull(r2);
assertFalse(r2.isEmpty());
// row2 should have a full put aggregating prior put + 10 increments
Cell r2Cell = r2.getColumnLatestCell(familyBytes, columnBytes);
assertNotNull(r2Cell);
assertEquals(20L, Bytes.toLong(r2Cell.getValue()));
// add 30 more increments to row2
for (int i = 0; i < 30; i++) {
Put p = new Put(row2);
p.add(familyBytes, columnBytes, ts++, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
}
// row2 should now have a full put aggregating prior 20 value + 30 increments
r2 = region.get(new Get(row2));
assertNotNull(r2);
assertFalse(r2.isEmpty());
r2Cell = r2.getColumnLatestCell(familyBytes, columnBytes);
assertNotNull(r2Cell);
assertEquals(50L, Bytes.toLong(r2Cell.getValue()));
// force another region flush
region.flushcache();
region.waitForFlushesAndCompactions();
// add 100 more increments to row2
for (int i = 0; i < 100; i++) {
Put p = new Put(row2);
p.add(familyBytes, columnBytes, ts++, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
}
// row2 should now have a full put aggregating prior 50 value + 100 increments
r2 = region.get(new Get(row2));
assertNotNull(r2);
assertFalse(r2.isEmpty());
r2Cell = r2.getColumnLatestCell(familyBytes, columnBytes);
assertNotNull(r2Cell);
assertEquals(150L, Bytes.toLong(r2Cell.getValue()));
} finally {
region.close();
}
}
use of co.cask.cdap.data2.util.TableId in project cdap by caskdata.
the class IncrementSummingScannerTest method testMultiColumnFlushAndCompact.
@Test
public void testMultiColumnFlushAndCompact() throws Exception {
TableId tableId = TableId.from(NamespaceId.DEFAULT.getNamespace(), "testMultiColumnFlushAndCompact");
byte[] familyBytes = Bytes.toBytes("f");
byte[] columnBytes = Bytes.toBytes("c");
byte[] columnBytes2 = Bytes.toBytes("c2");
HRegion region = createRegion(tableId, familyBytes);
try {
region.initialize();
long now = 1;
byte[] row1 = Bytes.toBytes("row1");
byte[] row2 = Bytes.toBytes("row2");
// Initial put to row1,c2
Put row1P = new Put(row1);
row1P.add(familyBytes, columnBytes2, now - 1, Bytes.toBytes(5L));
region.put(row1P);
// Initial put to row2,c
Put row2P = new Put(row2);
row2P.add(familyBytes, columnBytes, now - 1, Bytes.toBytes(10L));
region.put(row2P);
// Generate some increments
long ts = now;
for (int i = 0; i < 50; i++) {
region.put(generateIncrementPut(familyBytes, columnBytes, row1, ts));
region.put(generateIncrementPut(familyBytes, columnBytes, row2, ts));
region.put(generateIncrementPut(familyBytes, columnBytes2, row1, ts));
ts++;
}
// First scanner represents flush scanner
RegionScanner scanner = new IncrementSummingScanner(region, -1, region.getScanner(new Scan().setMaxVersions()), ScanType.COMPACT_RETAIN_DELETES, now + 15, -1);
// Second scanner is a user scan, this is to help in easy asserts
scanner = new IncrementSummingScanner(region, -1, scanner, ScanType.USER_SCAN);
List<Cell> results = Lists.newArrayList();
assertTrue(scanner.next(results, ScannerContext.newBuilder().setBatchLimit(10).build()));
assertEquals(2, results.size());
Cell cell = results.get(0);
assertNotNull(cell);
assertEquals("row1", Bytes.toString(cell.getRow()));
assertEquals("c", Bytes.toString(cell.getQualifier()));
assertEquals(50, Bytes.toLong(cell.getValue()));
cell = results.get(1);
assertNotNull(cell);
assertEquals("row1", Bytes.toString(cell.getRow()));
assertEquals("c2", Bytes.toString(cell.getQualifier()));
assertEquals(55, Bytes.toLong(cell.getValue()));
results.clear();
assertFalse(scanner.next(results, ScannerContext.newBuilder().setBatchLimit(10).build()));
assertEquals(1, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals("row2", Bytes.toString(cell.getRow()));
assertEquals(60, Bytes.toLong(cell.getValue()));
} finally {
region.close();
}
}
use of co.cask.cdap.data2.util.TableId in project cdap by caskdata.
the class IncrementSummingScannerTest method testIncrementScanningWithBatchAndUVB.
@Test
public void testIncrementScanningWithBatchAndUVB() throws Exception {
TableId tableId = TableId.from(NamespaceId.DEFAULT.getNamespace(), "TestIncrementSummingScannerWithUpperVisibilityBound");
byte[] familyBytes = Bytes.toBytes("f");
byte[] columnBytes = Bytes.toBytes("c");
HRegion region = createRegion(tableId, familyBytes);
try {
region.initialize();
long start = 0;
long now = start;
long counter1 = 0;
// adding 5 delta increments
for (int i = 0; i < 5; i++) {
Put p = new Put(Bytes.toBytes("r1"), now++);
p.add(familyBytes, columnBytes, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
counter1++;
}
// Also: we want different combinations of batch limit & uvbs
for (int i = 0; i < 7; i++) {
for (int k = 0; k < 4; k++) {
long[] uvbs = new long[k];
for (int l = 0; l < uvbs.length; l++) {
uvbs[l] = start + (k + 1) * (l + 1);
}
verifyCounts(region, new Scan().setMaxVersions(), new long[] { counter1 }, i > 0 ? i : -1, uvbs);
}
}
// Now test same with two groups of increments
int counter2 = 0;
for (int i = 0; i < 5; i++) {
Put p = new Put(Bytes.toBytes("r2"), now + i);
p.add(familyBytes, columnBytes, Bytes.toBytes(2L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
counter2 += 2;
}
for (int i = 0; i < 12; i++) {
for (int k = 0; k < 4; k++) {
long[] uvbs = new long[k];
for (int l = 0; l < uvbs.length; l++) {
uvbs[l] = start + (k + 1) * (l + 1);
}
verifyCounts(region, new Scan().setMaxVersions(), new long[] { counter1, counter2 }, i > 0 ? i : -1, uvbs);
}
}
} finally {
region.close();
}
}
use of co.cask.cdap.data2.util.TableId in project cdap by caskdata.
the class IncrementSummingScannerTest method testFlushAndCompact.
@Test
public void testFlushAndCompact() throws Exception {
TableId tableId = TableId.from(NamespaceId.DEFAULT.getNamespace(), "TestFlushAndCompact");
byte[] familyBytes = Bytes.toBytes("f");
byte[] columnBytes = Bytes.toBytes("c");
HRegion region = createRegion(tableId, familyBytes);
try {
region.initialize();
// load an initial set of increments
long ts = System.currentTimeMillis();
byte[] row1 = Bytes.toBytes("row1");
for (int i = 0; i < 50; i++) {
Put p = new Put(row1);
p.add(familyBytes, columnBytes, ts, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
ts++;
region.put(p);
}
byte[] row2 = Bytes.toBytes("row2");
ts = System.currentTimeMillis();
// start with a full put
Put row2P = new Put(row2);
row2P.add(familyBytes, columnBytes, ts++, Bytes.toBytes(10L));
region.put(row2P);
for (int i = 0; i < 10; i++) {
Put p = new Put(row2);
p.add(familyBytes, columnBytes, ts++, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
}
// force a region flush
region.flushcache();
region.waitForFlushesAndCompactions();
Result r1 = region.get(new Get(row1));
assertNotNull(r1);
assertFalse(r1.isEmpty());
// row1 should have a full put aggregating all 50 incrments
Cell r1Cell = r1.getColumnLatestCell(familyBytes, columnBytes);
assertNotNull(r1Cell);
assertEquals(50L, Bytes.toLong(r1Cell.getValue()));
Result r2 = region.get(new Get(row2));
assertNotNull(r2);
assertFalse(r2.isEmpty());
// row2 should have a full put aggregating prior put + 10 increments
Cell r2Cell = r2.getColumnLatestCell(familyBytes, columnBytes);
assertNotNull(r2Cell);
assertEquals(20L, Bytes.toLong(r2Cell.getValue()));
// add 30 more increments to row2
for (int i = 0; i < 30; i++) {
Put p = new Put(row2);
p.add(familyBytes, columnBytes, ts++, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
}
// row2 should now have a full put aggregating prior 20 value + 30 increments
r2 = region.get(new Get(row2));
assertNotNull(r2);
assertFalse(r2.isEmpty());
r2Cell = r2.getColumnLatestCell(familyBytes, columnBytes);
assertNotNull(r2Cell);
assertEquals(50L, Bytes.toLong(r2Cell.getValue()));
// force another region flush
region.flushcache();
region.waitForFlushesAndCompactions();
// add 100 more increments to row2
for (int i = 0; i < 100; i++) {
Put p = new Put(row2);
p.add(familyBytes, columnBytes, ts++, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
}
// row2 should now have a full put aggregating prior 50 value + 100 increments
r2 = region.get(new Get(row2));
assertNotNull(r2);
assertFalse(r2.isEmpty());
r2Cell = r2.getColumnLatestCell(familyBytes, columnBytes);
assertNotNull(r2Cell);
assertEquals(150L, Bytes.toLong(r2Cell.getValue()));
} finally {
region.close();
}
}
Aggregations