use of org.apache.hadoop.hbase.regionserver.HRegion in project hbase by apache.
the class TestDurability method testDurability.
@Test
public void testDurability() throws Exception {
final WALFactory wals = new WALFactory(CONF, null, ServerName.valueOf("TestDurability", 16010, System.currentTimeMillis()).toString());
byte[] tableName = Bytes.toBytes("TestDurability");
final WAL wal = wals.getWAL(tableName, null);
HRegion region = createHRegion(tableName, "region", wal, Durability.USE_DEFAULT);
HRegion deferredRegion = createHRegion(tableName, "deferredRegion", wal, Durability.ASYNC_WAL);
region.put(newPut(null));
verifyWALCount(wals, wal, 1);
// a put through the deferred table does not write to the wal immediately,
// but maybe has been successfully sync-ed by the underlying AsyncWriter +
// AsyncFlusher thread
deferredRegion.put(newPut(null));
// but will after we sync the wal
wal.sync();
verifyWALCount(wals, wal, 2);
// a put through a deferred table will be sync with the put sync'ed put
deferredRegion.put(newPut(null));
wal.sync();
verifyWALCount(wals, wal, 3);
region.put(newPut(null));
verifyWALCount(wals, wal, 4);
// a put through a deferred table will be sync with the put sync'ed put
deferredRegion.put(newPut(Durability.USE_DEFAULT));
wal.sync();
verifyWALCount(wals, wal, 5);
region.put(newPut(Durability.USE_DEFAULT));
verifyWALCount(wals, wal, 6);
// SKIP_WAL never writes to the wal
region.put(newPut(Durability.SKIP_WAL));
deferredRegion.put(newPut(Durability.SKIP_WAL));
verifyWALCount(wals, wal, 6);
wal.sync();
verifyWALCount(wals, wal, 6);
// Async overrides sync table default
region.put(newPut(Durability.ASYNC_WAL));
deferredRegion.put(newPut(Durability.ASYNC_WAL));
wal.sync();
verifyWALCount(wals, wal, 8);
// sync overrides async table default
region.put(newPut(Durability.SYNC_WAL));
deferredRegion.put(newPut(Durability.SYNC_WAL));
verifyWALCount(wals, wal, 10);
// fsync behaves like sync
region.put(newPut(Durability.FSYNC_WAL));
deferredRegion.put(newPut(Durability.FSYNC_WAL));
verifyWALCount(wals, wal, 12);
}
use of org.apache.hadoop.hbase.regionserver.HRegion in project phoenix by apache.
the class PhoenixIndexBuilder method executeAtomicOp.
@Override
public List<Mutation> executeAtomicOp(Increment inc) throws IOException {
byte[] opBytes = inc.getAttribute(ATOMIC_OP_ATTRIB);
if (opBytes == null) {
// Unexpected
return null;
}
inc.setAttribute(ATOMIC_OP_ATTRIB, null);
Put put = null;
Delete delete = null;
// We cannot neither use the time stamp in the Increment to set the Get time range
// nor set the Put/Delete time stamp and have this be atomic as HBase does not
// handle that. Though we disallow using ON DUPLICATE KEY clause when the
// CURRENT_SCN is set, we still may have a time stamp set as of when the table
// was resolved on the client side. We need to ignore this as well due to limitations
// in HBase, but this isn't too bad as the time will be very close the the current
// time anyway.
long ts = HConstants.LATEST_TIMESTAMP;
byte[] rowKey = inc.getRow();
final Get get = new Get(rowKey);
if (isDupKeyIgnore(opBytes)) {
get.setFilter(new FirstKeyOnlyFilter());
Result result = this.env.getRegion().get(get);
return result.isEmpty() ? convertIncrementToPutInSingletonList(inc) : Collections.<Mutation>emptyList();
}
ByteArrayInputStream stream = new ByteArrayInputStream(opBytes);
DataInputStream input = new DataInputStream(stream);
boolean skipFirstOp = input.readBoolean();
short repeat = input.readShort();
final int[] estimatedSizeHolder = { 0 };
List<Pair<PTable, List<Expression>>> operations = Lists.newArrayListWithExpectedSize(3);
while (true) {
ExpressionVisitor<Void> visitor = new StatelessTraverseAllExpressionVisitor<Void>() {
@Override
public Void visit(KeyValueColumnExpression expression) {
get.addColumn(expression.getColumnFamily(), expression.getColumnQualifier());
estimatedSizeHolder[0]++;
return null;
}
};
try {
int nExpressions = WritableUtils.readVInt(input);
List<Expression> expressions = Lists.newArrayListWithExpectedSize(nExpressions);
for (int i = 0; i < nExpressions; i++) {
Expression expression = ExpressionType.values()[WritableUtils.readVInt(input)].newInstance();
expression.readFields(input);
expressions.add(expression);
expression.accept(visitor);
}
PTableProtos.PTable tableProto = PTableProtos.PTable.parseDelimitedFrom(input);
PTable table = PTableImpl.createFromProto(tableProto);
operations.add(new Pair<>(table, expressions));
} catch (EOFException e) {
break;
}
}
int estimatedSize = estimatedSizeHolder[0];
if (get.getFamilyMap().isEmpty()) {
get.setFilter(new FirstKeyOnlyFilter());
}
MultiKeyValueTuple tuple;
List<Cell> flattenedCells = null;
List<Cell> cells = ((HRegion) this.env.getRegion()).get(get, false);
if (cells.isEmpty()) {
if (skipFirstOp) {
if (operations.size() <= 1 && repeat <= 1) {
return convertIncrementToPutInSingletonList(inc);
}
// Skip first operation (if first wasn't ON DUPLICATE KEY IGNORE)
repeat--;
}
// Base current state off of new row
flattenedCells = flattenCells(inc, estimatedSize);
tuple = new MultiKeyValueTuple(flattenedCells);
} else {
// Base current state off of existing row
tuple = new MultiKeyValueTuple(cells);
}
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
for (int opIndex = 0; opIndex < operations.size(); opIndex++) {
Pair<PTable, List<Expression>> operation = operations.get(opIndex);
PTable table = operation.getFirst();
List<Expression> expressions = operation.getSecond();
for (int j = 0; j < repeat; j++) {
// repeater loop
ptr.set(rowKey);
// executed, not when the outer loop is exited. Hence we do it here, at the top of the loop.
if (flattenedCells != null) {
Collections.sort(flattenedCells, KeyValue.COMPARATOR);
}
PRow row = table.newRow(GenericKeyValueBuilder.INSTANCE, ts, ptr, false);
for (int i = 0; i < expressions.size(); i++) {
Expression expression = expressions.get(i);
ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
expression.evaluate(tuple, ptr);
PColumn column = table.getColumns().get(i + 1);
Object value = expression.getDataType().toObject(ptr, column.getSortOrder());
// same type.
if (!column.getDataType().isSizeCompatible(ptr, value, column.getDataType(), expression.getSortOrder(), expression.getMaxLength(), expression.getScale(), column.getMaxLength(), column.getScale())) {
throw new DataExceedsCapacityException(column.getDataType(), column.getMaxLength(), column.getScale());
}
column.getDataType().coerceBytes(ptr, value, expression.getDataType(), expression.getMaxLength(), expression.getScale(), expression.getSortOrder(), column.getMaxLength(), column.getScale(), column.getSortOrder(), table.rowKeyOrderOptimizable());
byte[] bytes = ByteUtil.copyKeyBytesIfNecessary(ptr);
row.setValue(column, bytes);
}
flattenedCells = Lists.newArrayListWithExpectedSize(estimatedSize);
List<Mutation> mutations = row.toRowMutations();
for (Mutation source : mutations) {
flattenCells(source, flattenedCells);
}
tuple.setKeyValues(flattenedCells);
}
// Repeat only applies to first statement
repeat = 1;
}
List<Mutation> mutations = Lists.newArrayListWithExpectedSize(2);
for (int i = 0; i < tuple.size(); i++) {
Cell cell = tuple.getValue(i);
if (Type.codeToType(cell.getTypeByte()) == Type.Put) {
if (put == null) {
put = new Put(rowKey);
transferAttributes(inc, put);
mutations.add(put);
}
put.add(cell);
} else {
if (delete == null) {
delete = new Delete(rowKey);
transferAttributes(inc, delete);
mutations.add(delete);
}
delete.addDeleteMarker(cell);
}
}
return mutations;
}
use of org.apache.hadoop.hbase.regionserver.HRegion in project cdap by caskdata.
the class IncrementSummingScannerTest method testIncrementScanning.
@Test
public void testIncrementScanning() throws Exception {
TableId tableId = TableId.from(NamespaceId.DEFAULT.getNamespace(), "TestIncrementSummingScanner");
byte[] familyBytes = Bytes.toBytes("f");
byte[] columnBytes = Bytes.toBytes("c");
HRegion region = createRegion(tableId, familyBytes);
try {
region.initialize();
// test handling of a single increment value alone
Put p = new Put(Bytes.toBytes("r1"));
p.add(familyBytes, columnBytes, Bytes.toBytes(3L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
Scan scan = new Scan();
RegionScanner scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.USER_SCAN);
List<Cell> results = Lists.newArrayList();
scanner.next(results);
assertEquals(1, results.size());
Cell cell = results.get(0);
assertNotNull(cell);
assertEquals(3L, Bytes.toLong(cell.getValue()));
// test handling of a single total sum
p = new Put(Bytes.toBytes("r2"));
p.add(familyBytes, columnBytes, Bytes.toBytes(5L));
region.put(p);
scan = new Scan(Bytes.toBytes("r2"));
scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.USER_SCAN);
results = Lists.newArrayList();
scanner.next(results);
assertEquals(1, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals(5L, Bytes.toLong(cell.getValue()));
// test handling of multiple increment values
long now = System.currentTimeMillis();
p = new Put(Bytes.toBytes("r3"));
for (int i = 0; i < 5; i++) {
p.add(familyBytes, columnBytes, now - i, Bytes.toBytes((long) (i + 1)));
}
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
scan = new Scan(Bytes.toBytes("r3"));
scan.setMaxVersions();
scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.USER_SCAN);
results = Lists.newArrayList();
scanner.next(results);
assertEquals(1, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals(15L, Bytes.toLong(cell.getValue()));
// test handling of multiple increment values followed by a total sum, then other increments
now = System.currentTimeMillis();
p = new Put(Bytes.toBytes("r4"));
for (int i = 0; i < 3; i++) {
p.add(familyBytes, columnBytes, now - i, Bytes.toBytes(1L));
}
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
// this put will appear as a "total" sum prior to all the delta puts
p = new Put(Bytes.toBytes("r4"));
p.add(familyBytes, columnBytes, now - 5, Bytes.toBytes(5L));
region.put(p);
scan = new Scan(Bytes.toBytes("r4"));
scan.setMaxVersions();
scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.USER_SCAN);
results = Lists.newArrayList();
scanner.next(results);
assertEquals(1, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals(8L, Bytes.toLong(cell.getValue()));
// test handling of an increment column followed by a non-increment column
p = new Put(Bytes.toBytes("r4"));
p.add(familyBytes, Bytes.toBytes("c2"), Bytes.toBytes("value"));
region.put(p);
scan = new Scan(Bytes.toBytes("r4"));
scan.setMaxVersions();
scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.USER_SCAN);
results = Lists.newArrayList();
scanner.next(results);
assertEquals(2, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals(8L, Bytes.toLong(cell.getValue()));
cell = results.get(1);
assertNotNull(cell);
assertEquals("value", Bytes.toString(cell.getValue()));
// test handling of an increment column followed by a delete
now = System.currentTimeMillis();
Delete d = new Delete(Bytes.toBytes("r5"));
d.deleteColumn(familyBytes, columnBytes, now - 3);
region.delete(d);
p = new Put(Bytes.toBytes("r5"));
for (int i = 2; i >= 0; i--) {
p.add(familyBytes, columnBytes, now - i, Bytes.toBytes(1L));
}
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
scan = new Scan(Bytes.toBytes("r5"));
scan.setMaxVersions();
scan.setRaw(true);
scanner = new IncrementSummingScanner(region, -1, region.getScanner(scan), ScanType.COMPACT_RETAIN_DELETES);
results = Lists.newArrayList();
scanner.next(results);
// delete marker will not be returned for user scan
assertEquals(2, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals(3L, Bytes.toLong(cell.getValue(), IncrementHandlerState.DELTA_MAGIC_PREFIX.length, 8));
// next cell should be the delete
cell = results.get(1);
assertTrue(CellUtil.isDelete(cell));
} finally {
region.close();
}
}
use of org.apache.hadoop.hbase.regionserver.HRegion in project cdap by caskdata.
the class IncrementSummingScannerTest method testIncrementScanningWithBatchAndUVB.
@Test
public void testIncrementScanningWithBatchAndUVB() throws Exception {
TableId tableId = TableId.from(NamespaceId.DEFAULT.getNamespace(), "TestIncrementSummingScannerWithUpperVisibilityBound");
byte[] familyBytes = Bytes.toBytes("f");
byte[] columnBytes = Bytes.toBytes("c");
HRegion region = createRegion(tableId, familyBytes);
try {
region.initialize();
long start = 0;
long now = start;
long counter1 = 0;
// adding 5 delta increments
for (int i = 0; i < 5; i++) {
Put p = new Put(Bytes.toBytes("r1"), now++);
p.add(familyBytes, columnBytes, Bytes.toBytes(1L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
counter1++;
}
// Also: we want different combinations of batch limit & uvbs
for (int i = 0; i < 7; i++) {
for (int k = 0; k < 4; k++) {
long[] uvbs = new long[k];
for (int l = 0; l < uvbs.length; l++) {
uvbs[l] = start + (k + 1) * (l + 1);
}
verifyCounts(region, new Scan().setMaxVersions(), new long[] { counter1 }, i > 0 ? i : -1, uvbs);
}
}
// Now test same with two groups of increments
int counter2 = 0;
for (int i = 0; i < 5; i++) {
Put p = new Put(Bytes.toBytes("r2"), now + i);
p.add(familyBytes, columnBytes, Bytes.toBytes(2L));
p.setAttribute(HBaseTable.DELTA_WRITE, TRUE);
region.put(p);
counter2 += 2;
}
for (int i = 0; i < 12; i++) {
for (int k = 0; k < 4; k++) {
long[] uvbs = new long[k];
for (int l = 0; l < uvbs.length; l++) {
uvbs[l] = start + (k + 1) * (l + 1);
}
verifyCounts(region, new Scan().setMaxVersions(), new long[] { counter1, counter2 }, i > 0 ? i : -1, uvbs);
}
}
} finally {
region.close();
}
}
use of org.apache.hadoop.hbase.regionserver.HRegion in project cdap by caskdata.
the class IncrementSummingScannerTest method testMultiColumnFlushAndCompact.
@Test
public void testMultiColumnFlushAndCompact() throws Exception {
TableId tableId = TableId.from(NamespaceId.DEFAULT.getNamespace(), "testMultiColumnFlushAndCompact");
byte[] familyBytes = Bytes.toBytes("f");
byte[] columnBytes = Bytes.toBytes("c");
byte[] columnBytes2 = Bytes.toBytes("c2");
HRegion region = createRegion(tableId, familyBytes);
try {
region.initialize();
long now = 1;
byte[] row1 = Bytes.toBytes("row1");
byte[] row2 = Bytes.toBytes("row2");
// Initial put to row1,c2
Put row1P = new Put(row1);
row1P.add(familyBytes, columnBytes2, now - 1, Bytes.toBytes(5L));
region.put(row1P);
// Initial put to row2,c
Put row2P = new Put(row2);
row2P.add(familyBytes, columnBytes, now - 1, Bytes.toBytes(10L));
region.put(row2P);
// Generate some increments
long ts = now;
for (int i = 0; i < 50; i++) {
region.put(generateIncrementPut(familyBytes, columnBytes, row1, ts));
region.put(generateIncrementPut(familyBytes, columnBytes, row2, ts));
region.put(generateIncrementPut(familyBytes, columnBytes2, row1, ts));
ts++;
}
// First scanner represents flush scanner
RegionScanner scanner = new IncrementSummingScanner(region, -1, region.getScanner(new Scan().setMaxVersions()), ScanType.COMPACT_RETAIN_DELETES, now + 15, -1);
// Second scanner is a user scan, this is to help in easy asserts
scanner = new IncrementSummingScanner(region, -1, scanner, ScanType.USER_SCAN);
List<Cell> results = Lists.newArrayList();
assertTrue(scanner.next(results, ScannerContext.newBuilder().setBatchLimit(10).build()));
assertEquals(2, results.size());
Cell cell = results.get(0);
assertNotNull(cell);
assertEquals("row1", Bytes.toString(cell.getRow()));
assertEquals("c", Bytes.toString(cell.getQualifier()));
assertEquals(50, Bytes.toLong(cell.getValue()));
cell = results.get(1);
assertNotNull(cell);
assertEquals("row1", Bytes.toString(cell.getRow()));
assertEquals("c2", Bytes.toString(cell.getQualifier()));
assertEquals(55, Bytes.toLong(cell.getValue()));
results.clear();
assertFalse(scanner.next(results, ScannerContext.newBuilder().setBatchLimit(10).build()));
assertEquals(1, results.size());
cell = results.get(0);
assertNotNull(cell);
assertEquals("row2", Bytes.toString(cell.getRow()));
assertEquals(60, Bytes.toLong(cell.getValue()));
} finally {
region.close();
}
}
Aggregations