use of io.cdap.cdap.api.dataset.table.Get in project cdap by cdapio.
the class BufferingTable method getPersisted.
/**
* Fetches a list of rows from persistent store. Subclasses should override this if they can batch multiple
* gets into a single request, as the default implementation simply loops through the gets and calls
* {@link #getPersisted(byte[], byte[][])} on each get.
* NOTE: persisted store can also be in-memory, it is called "persisted" to distinguish from in-memory buffer.
* @param gets list of gets to perform
* @return list of rows, one for each get
* @throws Exception
*/
protected List<Map<byte[], byte[]>> getPersisted(List<Get> gets) throws Exception {
List<Map<byte[], byte[]>> results = Lists.newArrayListWithCapacity(gets.size());
for (Get get : gets) {
List<byte[]> getColumns = get.getColumns();
byte[][] columns = getColumns == null ? null : getColumns.toArray(new byte[getColumns.size()][]);
results.add(getPersisted(get.getRow(), columns));
}
return results;
}
use of io.cdap.cdap.api.dataset.table.Get in project cdap by cdapio.
the class BufferingTable method get.
@ReadOnly
@Override
public List<Row> get(List<Get> gets) {
ensureTransactionIsStarted();
try {
// get persisted, then overwrite with whats buffered
List<Map<byte[], byte[]>> persistedRows = getPersisted(gets);
// gets and rows lists are always of the same size
Preconditions.checkArgument(gets.size() == persistedRows.size(), "Invalid number of rows fetched when performing multi-get. There must be one row for each get.");
List<Row> result = Lists.newArrayListWithCapacity(persistedRows.size());
Iterator<Map<byte[], byte[]>> persistedRowsIter = persistedRows.iterator();
Iterator<Get> getIter = gets.iterator();
while (persistedRowsIter.hasNext() && getIter.hasNext()) {
Get get = getIter.next();
Map<byte[], byte[]> persistedRow = persistedRowsIter.next();
// navigable copy of the persisted data. Implementation may return immutable or unmodifiable maps,
// so we make a copy here.
NavigableMap<byte[], byte[]> rowColumns = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
rowColumns.putAll(persistedRow);
byte[] row = get.getRow();
NavigableMap<byte[], Update> buffCols = buff.get(row);
// merge what was in the buffer and what was persisted
if (buffCols != null) {
List<byte[]> getColumns = get.getColumns();
byte[][] columns = getColumns == null ? null : getColumns.toArray(new byte[getColumns.size()][]);
mergeToPersisted(rowColumns, buffCols, columns);
}
result.add(new Result(row, unwrapDeletes(rowColumns)));
}
return result;
} catch (Exception e) {
LOG.debug("multi-get failed for table: " + getTransactionAwareName(), e);
throw new DataSetException("multi-get failed", e);
}
}
use of io.cdap.cdap.api.dataset.table.Get in project cdap by cdapio.
the class IndexedTableTest method testIndexedOperations.
@Test
public void testIndexedOperations() throws Exception {
TransactionExecutor txnl = dsFrameworkUtil.newTransactionExecutor(table);
// start a new transaction
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
// add a value c with idx = 1, and b with idx = 2
table.put(new Put(keyC).add(idxCol, idx1).add(valCol, valC));
table.put(new Put(keyB).add(idxCol, idx2).add(valCol, valB));
}
});
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
// read by key c
Row row = table.get(new Get(keyC, colIdxVal));
TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx1, valC });
// read by key b
row = table.get(new Get(keyB, colIdxVal));
TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx2, valB });
// read by idx 1 -> c
row = readFirst(table.readByIndex(idxCol, idx1));
TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx1, valC });
// read by idx 2 -> b
row = readFirst(table.readByIndex(idxCol, idx2));
TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx2, valB });
// test read over empty index (idx 3)
assertEmpty(table.readByIndex(idxCol, idx3));
}
});
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
// add a value a with idx = 1
table.put(new Put(keyA).add(idxCol, idx1).add(valCol, valA));
}
});
// read by idx 1 -> a
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
Row row = readFirst(table.readByIndex(idxCol, idx1));
TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx1, valA });
}
});
// start a new transaction
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
// delete value a
table.delete(new Delete(keyA, colIdxVal));
}
});
// read by idx 1 -> c
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
Row row = readFirst(table.readByIndex(idxCol, idx1));
TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx1, valC });
}
});
// start a new transaction
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
// add a value aa with idx 2
table.put(new Put(keyAA).add(idxCol, idx2).add(valCol, valAA));
}
});
// read by idx 2 -> aa
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
Row row = readFirst(table.readByIndex(idxCol, idx2));
TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx2, valAA });
}
});
// start a new transaction
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
// swap value for aa to ab
Assert.assertTrue(table.compareAndSwap(keyAA, valCol, valAA, valAB));
}
});
// read by idx 2 -> ab
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
Row row = readFirst(table.readByIndex(idxCol, idx2));
TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx2, valAB });
}
});
// start a new transaction
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
// swap value for aa to bb
Assert.assertTrue(table.compareAndSwap(keyAA, valCol, valAB, valBB));
}
});
// read by idx 2 -> bb (value of key aa)
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
Row row = readFirst(table.readByIndex(idxCol, idx2));
TableAssert.assertColumns(row, colIdxVal, new byte[][] { idx2, valBB });
}
});
// start a new transaction
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
// swap value for aa to null
Assert.assertTrue(table.compareAndSwap(keyAA, valCol, valBB, null));
}
});
// read by idx 2 -> null (value of b)
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
Row row = readFirst(table.readByIndex(idxCol, idx2));
TableAssert.assertColumn(row, idxCol, idx2);
}
});
// start a new transaction
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
// swap idx for c to 3
Assert.assertTrue(table.compareAndSwap(keyC, idxCol, idx1, idx3));
}
});
// read by idx 1 -> null (no row has that any more)
txnl.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
assertEmpty(table.readByIndex(idxCol, idx1));
// read by idx 3 > c
Row row = readFirst(table.readByIndex(idxCol, idx3));
TableAssert.assertColumns(row, new byte[][] { idxCol, valCol }, new byte[][] { idx3, valC });
}
});
}
use of io.cdap.cdap.api.dataset.table.Get in project cdap by cdapio.
the class TestFrameworkTestRun method testAppWithTxTimeout.
// CDAP-18061 opened for tacking fix of this flaky test case
@Ignore
@Category(SlowTests.class)
@Test
public void testAppWithTxTimeout() throws Exception {
int txDefaulTimeoutService = 17;
int txDefaulTimeoutWorker = 18;
int txDefaulTimeoutWorkflow = 19;
int txDefaulTimeoutAction = 20;
int txDefaulTimeoutMapReduce = 23;
int txDefaulTimeoutSpark = 24;
final ApplicationManager appManager = deployApplication(testSpace, AppWithCustomTx.class);
try {
// attempt to start with a tx timeout that exceeds the max tx timeout
appManager.getServiceManager(AppWithCustomTx.SERVICE).start(txTimeoutArguments(100000));
// wait for the failed status of AppWithCustomTx.SERVICE to be persisted, so that it can be started again
Tasks.waitFor(1, () -> appManager.getServiceManager(AppWithCustomTx.SERVICE).getHistory(ProgramRunStatus.FAILED).size(), 30L, TimeUnit.SECONDS, 1, TimeUnit.SECONDS);
ServiceManager serviceManager = appManager.getServiceManager(AppWithCustomTx.SERVICE).start(txTimeoutArguments(txDefaulTimeoutService));
WorkerManager notxWorkerManager = appManager.getWorkerManager(AppWithCustomTx.WORKER_NOTX).start(txTimeoutArguments(txDefaulTimeoutWorker));
WorkerManager txWorkerManager = appManager.getWorkerManager(AppWithCustomTx.WORKER_TX).start(txTimeoutArguments(txDefaulTimeoutWorker));
WorkflowManager txWFManager = appManager.getWorkflowManager(AppWithCustomTx.WORKFLOW_TX).start(txTimeoutArguments(txDefaulTimeoutWorkflow));
WorkflowManager notxWFManager = appManager.getWorkflowManager(AppWithCustomTx.WORKFLOW_NOTX).start(txTimeoutArguments(txDefaulTimeoutWorkflow, txDefaulTimeoutAction, "action", AppWithCustomTx.ACTION_NOTX));
MapReduceManager txMRManager = appManager.getMapReduceManager(AppWithCustomTx.MAPREDUCE_TX).start(txTimeoutArguments(txDefaulTimeoutMapReduce));
MapReduceManager notxMRManager = appManager.getMapReduceManager(AppWithCustomTx.MAPREDUCE_NOTX).start(txTimeoutArguments(txDefaulTimeoutMapReduce));
SparkManager txSparkManager = appManager.getSparkManager(AppWithCustomTx.SPARK_TX).start(txTimeoutArguments(txDefaulTimeoutSpark));
SparkManager notxSparkManager = appManager.getSparkManager(AppWithCustomTx.SPARK_NOTX).start(txTimeoutArguments(txDefaulTimeoutSpark));
serviceManager.waitForRun(ProgramRunStatus.RUNNING, 10, TimeUnit.SECONDS);
callServicePut(serviceManager.getServiceURL(), "tx", "hello");
callServicePut(serviceManager.getServiceURL(), "tx", AppWithCustomTx.FAIL_PRODUCER, 200);
callServicePut(serviceManager.getServiceURL(), "tx", AppWithCustomTx.FAIL_CONSUMER, 500);
callServicePut(serviceManager.getServiceURL(), "notx", "hello");
callServicePut(serviceManager.getServiceURL(), "notx", AppWithCustomTx.FAIL_PRODUCER, 200);
callServicePut(serviceManager.getServiceURL(), "notx", AppWithCustomTx.FAIL_CONSUMER, 500);
serviceManager.stop();
serviceManager.waitForRun(ProgramRunStatus.KILLED, 10, TimeUnit.SECONDS);
txMRManager.waitForRun(ProgramRunStatus.FAILED, 10L, TimeUnit.SECONDS);
notxMRManager.waitForRun(ProgramRunStatus.FAILED, 10L, TimeUnit.SECONDS);
txSparkManager.waitForRun(ProgramRunStatus.COMPLETED, 10L, TimeUnit.SECONDS);
notxSparkManager.waitForRun(ProgramRunStatus.COMPLETED, 10L, TimeUnit.SECONDS);
notxWorkerManager.waitForRun(ProgramRunStatus.COMPLETED, 10L, TimeUnit.SECONDS);
txWorkerManager.waitForRun(ProgramRunStatus.COMPLETED, 10L, TimeUnit.SECONDS);
txWFManager.waitForRun(ProgramRunStatus.COMPLETED, 10L, TimeUnit.SECONDS);
notxWFManager.waitForRun(ProgramRunStatus.COMPLETED, 10L, TimeUnit.SECONDS);
DataSetManager<TransactionCapturingTable> dataset = getDataset(testSpace.dataset(AppWithCustomTx.CAPTURE));
Table t = dataset.get().getTable();
// all programs attempt to write to the table in different transactional contexts
// - if it is outside a transaction, then the expected value is null
// - if it is in an attempt of a nested transaction, then the expected value is null
// - if it is in an implicit transaction, then the expected value is "default"
// - if it is in an explicit transaction, then the expected value is the transaction timeout
Object[][] writesToValidate = new Object[][] { // transactions attempted by the workers
{ AppWithCustomTx.WORKER_TX, AppWithCustomTx.INITIALIZE, txDefaulTimeoutWorker }, { AppWithCustomTx.WORKER_TX, AppWithCustomTx.INITIALIZE_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.WORKER_TX, AppWithCustomTx.DESTROY, txDefaulTimeoutWorker }, { AppWithCustomTx.WORKER_TX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.WORKER_TX, AppWithCustomTx.RUNTIME, null }, { AppWithCustomTx.WORKER_TX, AppWithCustomTx.RUNTIME_TX_D, txDefaulTimeoutWorker }, { AppWithCustomTx.WORKER_TX, AppWithCustomTx.RUNTIME_TX, AppWithCustomTx.TIMEOUT_WORKER_RUNTIME }, { AppWithCustomTx.WORKER_TX, AppWithCustomTx.RUNTIME_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.WORKER_NOTX, AppWithCustomTx.INITIALIZE, null }, { AppWithCustomTx.WORKER_NOTX, AppWithCustomTx.INITIALIZE_TX, AppWithCustomTx.TIMEOUT_WORKER_INITIALIZE }, { AppWithCustomTx.WORKER_NOTX, AppWithCustomTx.INITIALIZE_TX_D, txDefaulTimeoutWorker }, { AppWithCustomTx.WORKER_NOTX, AppWithCustomTx.INITIALIZE_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.WORKER_NOTX, AppWithCustomTx.DESTROY, null }, { AppWithCustomTx.WORKER_NOTX, AppWithCustomTx.DESTROY_TX, AppWithCustomTx.TIMEOUT_WORKER_DESTROY }, { AppWithCustomTx.WORKER_NOTX, AppWithCustomTx.DESTROY_TX_D, txDefaulTimeoutWorker }, { AppWithCustomTx.WORKER_NOTX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.WORKER_NOTX, AppWithCustomTx.RUNTIME, null }, { AppWithCustomTx.WORKER_NOTX, AppWithCustomTx.RUNTIME_TX, AppWithCustomTx.TIMEOUT_WORKER_RUNTIME }, { AppWithCustomTx.WORKER_NOTX, AppWithCustomTx.RUNTIME_TX_D, txDefaulTimeoutWorker }, { AppWithCustomTx.WORKER_NOTX, AppWithCustomTx.RUNTIME_NEST, AppWithCustomTx.FAILED }, // transactions attempted by the service
{ AppWithCustomTx.HANDLER_TX, AppWithCustomTx.INITIALIZE, txDefaulTimeoutService }, { AppWithCustomTx.HANDLER_TX, AppWithCustomTx.INITIALIZE_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.HANDLER_TX, AppWithCustomTx.DESTROY, txDefaulTimeoutService }, { AppWithCustomTx.HANDLER_TX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.HANDLER_TX, AppWithCustomTx.RUNTIME, txDefaulTimeoutService }, { AppWithCustomTx.HANDLER_TX, AppWithCustomTx.RUNTIME_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.CONSUMER_TX, AppWithCustomTx.RUNTIME, null }, { AppWithCustomTx.CONSUMER_TX, AppWithCustomTx.RUNTIME_TX, AppWithCustomTx.TIMEOUT_CONSUMER_RUNTIME }, { AppWithCustomTx.CONSUMER_TX, AppWithCustomTx.RUNTIME_TX_D, txDefaulTimeoutService }, { AppWithCustomTx.CONSUMER_TX, AppWithCustomTx.RUNTIME_TX_T, AppWithCustomTx.TIMEOUT_CONSUMER_RUNTIME }, { AppWithCustomTx.CONSUMER_TX, AppWithCustomTx.RUNTIME_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.CONSUMER_TX, AppWithCustomTx.RUNTIME_NEST_T, AppWithCustomTx.FAILED }, { AppWithCustomTx.CONSUMER_TX, AppWithCustomTx.RUNTIME_NEST_CT, AppWithCustomTx.FAILED }, { AppWithCustomTx.CONSUMER_TX, AppWithCustomTx.RUNTIME_NEST_TC, AppWithCustomTx.FAILED }, { AppWithCustomTx.CONSUMER_TX, AppWithCustomTx.DESTROY, txDefaulTimeoutService }, { AppWithCustomTx.CONSUMER_TX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.CONSUMER_TX, AppWithCustomTx.ONERROR, txDefaulTimeoutService }, { AppWithCustomTx.CONSUMER_TX, AppWithCustomTx.ONERROR_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.PRODUCER_TX, AppWithCustomTx.RUNTIME, null }, { AppWithCustomTx.PRODUCER_TX, AppWithCustomTx.RUNTIME_TX, AppWithCustomTx.TIMEOUT_PRODUCER_RUNTIME }, { AppWithCustomTx.PRODUCER_TX, AppWithCustomTx.RUNTIME_TX_D, txDefaulTimeoutService }, { AppWithCustomTx.PRODUCER_TX, AppWithCustomTx.RUNTIME_TX_T, AppWithCustomTx.TIMEOUT_PRODUCER_RUNTIME }, { AppWithCustomTx.PRODUCER_TX, AppWithCustomTx.RUNTIME_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.PRODUCER_TX, AppWithCustomTx.RUNTIME_NEST_T, AppWithCustomTx.FAILED }, { AppWithCustomTx.PRODUCER_TX, AppWithCustomTx.RUNTIME_NEST_CT, AppWithCustomTx.FAILED }, { AppWithCustomTx.PRODUCER_TX, AppWithCustomTx.RUNTIME_NEST_TC, AppWithCustomTx.FAILED }, // { AppWithCustomTx.PRODUCER_TX, AppWithCustomTx.DESTROY, txDefaulTimeoutService },
{ AppWithCustomTx.PRODUCER_TX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.PRODUCER_TX, AppWithCustomTx.ONERROR, txDefaulTimeoutService }, { AppWithCustomTx.PRODUCER_TX, AppWithCustomTx.ONERROR_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.HANDLER_NOTX, AppWithCustomTx.INITIALIZE, null }, { AppWithCustomTx.HANDLER_NOTX, AppWithCustomTx.INITIALIZE_TX, AppWithCustomTx.TIMEOUT_HANDLER_INITIALIZE }, { AppWithCustomTx.HANDLER_NOTX, AppWithCustomTx.INITIALIZE_TX_D, txDefaulTimeoutService }, { AppWithCustomTx.HANDLER_NOTX, AppWithCustomTx.INITIALIZE_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.HANDLER_NOTX, AppWithCustomTx.DESTROY, null }, { AppWithCustomTx.HANDLER_NOTX, AppWithCustomTx.DESTROY_TX, AppWithCustomTx.TIMEOUT_HANDLER_DESTROY }, { AppWithCustomTx.HANDLER_NOTX, AppWithCustomTx.DESTROY_TX_D, txDefaulTimeoutService }, { AppWithCustomTx.HANDLER_NOTX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.HANDLER_NOTX, AppWithCustomTx.RUNTIME, null }, { AppWithCustomTx.HANDLER_NOTX, AppWithCustomTx.RUNTIME_TX, AppWithCustomTx.TIMEOUT_HANDLER_RUNTIME }, { AppWithCustomTx.HANDLER_NOTX, AppWithCustomTx.RUNTIME_TX_D, txDefaulTimeoutService }, { AppWithCustomTx.HANDLER_NOTX, AppWithCustomTx.RUNTIME_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.RUNTIME, null }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.RUNTIME_TX, AppWithCustomTx.TIMEOUT_CONSUMER_RUNTIME }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.RUNTIME_TX_D, txDefaulTimeoutService }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.RUNTIME_TX_T, AppWithCustomTx.TIMEOUT_CONSUMER_RUNTIME }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.RUNTIME_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.RUNTIME_NEST_T, AppWithCustomTx.FAILED }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.RUNTIME_NEST_CT, AppWithCustomTx.FAILED }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.RUNTIME_NEST_TC, AppWithCustomTx.FAILED }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.DESTROY, null }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.DESTROY_TX, AppWithCustomTx.TIMEOUT_CONSUMER_DESTROY }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.DESTROY_TX_D, txDefaulTimeoutService }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.ONERROR, null }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.ONERROR_TX, AppWithCustomTx.TIMEOUT_CONSUMER_ERROR }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.ONERROR_TX_D, txDefaulTimeoutService }, { AppWithCustomTx.CONSUMER_NOTX, AppWithCustomTx.ONERROR_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.RUNTIME, null }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.RUNTIME_TX, AppWithCustomTx.TIMEOUT_PRODUCER_RUNTIME }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.RUNTIME_TX_D, txDefaulTimeoutService }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.RUNTIME_TX_T, AppWithCustomTx.TIMEOUT_PRODUCER_RUNTIME }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.RUNTIME_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.RUNTIME_NEST_T, AppWithCustomTx.FAILED }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.RUNTIME_NEST_CT, AppWithCustomTx.FAILED }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.RUNTIME_NEST_TC, AppWithCustomTx.FAILED }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.DESTROY, null }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.DESTROY_TX, AppWithCustomTx.TIMEOUT_PRODUCER_DESTROY }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.DESTROY_TX_D, txDefaulTimeoutService }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.ONERROR, null }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.ONERROR_TX, AppWithCustomTx.TIMEOUT_PRODUCER_ERROR }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.ONERROR_TX_D, txDefaulTimeoutService }, { AppWithCustomTx.PRODUCER_NOTX, AppWithCustomTx.ONERROR_NEST, AppWithCustomTx.FAILED }, // transactions attempted by the workflows
{ AppWithCustomTx.WORKFLOW_TX, AppWithCustomTx.INITIALIZE, txDefaulTimeoutWorkflow }, { AppWithCustomTx.WORKFLOW_TX, AppWithCustomTx.INITIALIZE_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.WORKFLOW_TX, AppWithCustomTx.DESTROY, txDefaulTimeoutWorkflow }, { AppWithCustomTx.WORKFLOW_TX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.ACTION_TX, AppWithCustomTx.INITIALIZE, txDefaulTimeoutWorkflow }, { AppWithCustomTx.ACTION_TX, AppWithCustomTx.INITIALIZE_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.ACTION_TX, AppWithCustomTx.RUNTIME, null }, { AppWithCustomTx.ACTION_TX, AppWithCustomTx.RUNTIME_TX, AppWithCustomTx.TIMEOUT_ACTION_RUNTIME }, { AppWithCustomTx.ACTION_TX, AppWithCustomTx.RUNTIME_TX_D, txDefaulTimeoutWorkflow }, { AppWithCustomTx.ACTION_TX, AppWithCustomTx.RUNTIME_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.ACTION_TX, AppWithCustomTx.DESTROY, txDefaulTimeoutWorkflow }, { AppWithCustomTx.ACTION_TX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.WORKFLOW_NOTX, AppWithCustomTx.INITIALIZE, null }, { AppWithCustomTx.WORKFLOW_NOTX, AppWithCustomTx.INITIALIZE_TX, AppWithCustomTx.TIMEOUT_WORKFLOW_INITIALIZE }, { AppWithCustomTx.WORKFLOW_NOTX, AppWithCustomTx.INITIALIZE_TX_D, txDefaulTimeoutWorkflow }, { AppWithCustomTx.WORKFLOW_NOTX, AppWithCustomTx.INITIALIZE_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.WORKFLOW_NOTX, AppWithCustomTx.DESTROY, null }, { AppWithCustomTx.WORKFLOW_NOTX, AppWithCustomTx.DESTROY_TX, AppWithCustomTx.TIMEOUT_WORKFLOW_DESTROY }, { AppWithCustomTx.WORKFLOW_NOTX, AppWithCustomTx.DESTROY_TX_D, txDefaulTimeoutWorkflow }, { AppWithCustomTx.WORKFLOW_NOTX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.ACTION_NOTX, AppWithCustomTx.INITIALIZE, null }, { AppWithCustomTx.ACTION_NOTX, AppWithCustomTx.INITIALIZE_TX, AppWithCustomTx.TIMEOUT_ACTION_INITIALIZE }, { AppWithCustomTx.ACTION_NOTX, AppWithCustomTx.INITIALIZE_TX_D, txDefaulTimeoutAction }, { AppWithCustomTx.ACTION_NOTX, AppWithCustomTx.INITIALIZE_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.ACTION_NOTX, AppWithCustomTx.RUNTIME, null }, { AppWithCustomTx.ACTION_NOTX, AppWithCustomTx.RUNTIME_TX, AppWithCustomTx.TIMEOUT_ACTION_RUNTIME }, { AppWithCustomTx.ACTION_NOTX, AppWithCustomTx.RUNTIME_TX_D, txDefaulTimeoutAction }, { AppWithCustomTx.ACTION_NOTX, AppWithCustomTx.RUNTIME_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.ACTION_NOTX, AppWithCustomTx.DESTROY, null }, { AppWithCustomTx.ACTION_NOTX, AppWithCustomTx.DESTROY_TX, AppWithCustomTx.TIMEOUT_ACTION_DESTROY }, { AppWithCustomTx.ACTION_NOTX, AppWithCustomTx.DESTROY_TX_D, txDefaulTimeoutAction }, { AppWithCustomTx.ACTION_NOTX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED }, // transactions attempted by the mapreduce's
{ AppWithCustomTx.MAPREDUCE_TX, AppWithCustomTx.INITIALIZE, txDefaulTimeoutMapReduce }, { AppWithCustomTx.MAPREDUCE_TX, AppWithCustomTx.INITIALIZE_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.MAPREDUCE_TX, AppWithCustomTx.DESTROY, txDefaulTimeoutMapReduce }, { AppWithCustomTx.MAPREDUCE_TX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.MAPREDUCE_NOTX, AppWithCustomTx.INITIALIZE, null }, { AppWithCustomTx.MAPREDUCE_NOTX, AppWithCustomTx.INITIALIZE_TX, AppWithCustomTx.TIMEOUT_MAPREDUCE_INITIALIZE }, { AppWithCustomTx.MAPREDUCE_NOTX, AppWithCustomTx.INITIALIZE_TX_D, txDefaulTimeoutMapReduce }, { AppWithCustomTx.MAPREDUCE_NOTX, AppWithCustomTx.INITIALIZE_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.MAPREDUCE_NOTX, AppWithCustomTx.DESTROY, null }, { AppWithCustomTx.MAPREDUCE_NOTX, AppWithCustomTx.DESTROY_TX, AppWithCustomTx.TIMEOUT_MAPREDUCE_DESTROY }, { AppWithCustomTx.MAPREDUCE_NOTX, AppWithCustomTx.DESTROY_TX_D, txDefaulTimeoutMapReduce }, { AppWithCustomTx.MAPREDUCE_NOTX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED }, // transactions attempted by the spark's
{ AppWithCustomTx.SPARK_TX, AppWithCustomTx.INITIALIZE, txDefaulTimeoutSpark }, { AppWithCustomTx.SPARK_TX, AppWithCustomTx.INITIALIZE_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.SPARK_TX, AppWithCustomTx.DESTROY, txDefaulTimeoutSpark }, { AppWithCustomTx.SPARK_TX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.SPARK_NOTX, AppWithCustomTx.INITIALIZE, null }, { AppWithCustomTx.SPARK_NOTX, AppWithCustomTx.INITIALIZE_TX, AppWithCustomTx.TIMEOUT_SPARK_INITIALIZE }, { AppWithCustomTx.SPARK_NOTX, AppWithCustomTx.INITIALIZE_TX_D, txDefaulTimeoutSpark }, { AppWithCustomTx.SPARK_NOTX, AppWithCustomTx.INITIALIZE_NEST, AppWithCustomTx.FAILED }, { AppWithCustomTx.SPARK_NOTX, AppWithCustomTx.DESTROY, null }, { AppWithCustomTx.SPARK_NOTX, AppWithCustomTx.DESTROY_TX, AppWithCustomTx.TIMEOUT_SPARK_DESTROY }, { AppWithCustomTx.SPARK_NOTX, AppWithCustomTx.DESTROY_TX_D, txDefaulTimeoutSpark }, { AppWithCustomTx.SPARK_NOTX, AppWithCustomTx.DESTROY_NEST, AppWithCustomTx.FAILED } };
for (Object[] writeToValidate : writesToValidate) {
String row = (String) writeToValidate[0];
String column = (String) writeToValidate[1];
String expectedValue = writeToValidate[2] == null ? null : String.valueOf(writeToValidate[2]);
Tasks.waitFor(expectedValue, () -> t.get(new Get(row, column)).getString(column), 30L, TimeUnit.SECONDS, 1, TimeUnit.SECONDS, String.format("Error getting value for %s.%s. Expected: %s, Got: %s", row, column, expectedValue, t.get(new Get(row, column)).getString(column)));
}
} finally {
appManager.stopAll();
}
}
use of io.cdap.cdap.api.dataset.table.Get in project cdap by cdapio.
the class HBaseTableTest method testCachedEncodedTransaction.
@Test
public void testCachedEncodedTransaction() throws Exception {
String tableName = "testEncodedTxTable";
DatasetProperties props = DatasetProperties.EMPTY;
getTableAdmin(CONTEXT1, tableName, props).create();
DatasetSpecification tableSpec = DatasetSpecification.builder(tableName, HBaseTable.class.getName()).build();
// use a transaction codec that counts the number of times encode() is called
final AtomicInteger encodeCount = new AtomicInteger();
final TransactionCodec codec = new TransactionCodec() {
@Override
public byte[] encode(Transaction tx) throws IOException {
encodeCount.incrementAndGet();
return super.encode(tx);
}
};
// use a table util that creates an HTable that validates the encoded tx on each get
final AtomicReference<Transaction> txRef = new AtomicReference<>();
HBaseTableUtil util = new DelegatingHBaseTableUtil(hBaseTableUtil) {
@Override
public org.apache.hadoop.hbase.client.Table createTable(Configuration conf, TableId tableId) throws IOException {
org.apache.hadoop.hbase.client.Table table = super.createTable(conf, tableId);
return new DelegatingTable(table) {
@Override
public Result get(org.apache.hadoop.hbase.client.Get get) throws IOException {
Assert.assertEquals(txRef.get().getTransactionId(), codec.decode(get.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY)).getTransactionId());
return super.get(get);
}
@Override
public Result[] get(List<org.apache.hadoop.hbase.client.Get> gets) throws IOException {
for (org.apache.hadoop.hbase.client.Get get : gets) {
Assert.assertEquals(txRef.get().getTransactionId(), codec.decode(get.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY)).getTransactionId());
}
return super.get(gets);
}
@Override
public ResultScanner getScanner(org.apache.hadoop.hbase.client.Scan scan) throws IOException {
Assert.assertEquals(txRef.get().getTransactionId(), codec.decode(scan.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY)).getTransactionId());
return super.getScanner(scan);
}
};
}
};
HBaseTable table = new HBaseTable(CONTEXT1, tableSpec, Collections.<String, String>emptyMap(), cConf, TEST_HBASE.getConfiguration(), util, codec);
DetachedTxSystemClient txSystemClient = new DetachedTxSystemClient();
// test all operations: only the first one encodes
Transaction tx = txSystemClient.startShort();
txRef.set(tx);
table.startTx(tx);
table.put(b("row1"), b("col1"), b("val1"));
Assert.assertEquals(0, encodeCount.get());
table.get(b("row"));
Assert.assertEquals(1, encodeCount.get());
table.get(ImmutableList.of(new Get("a"), new Get("b")));
Assert.assertEquals(1, encodeCount.get());
Scanner scanner = table.scan(new Scan(null, null));
Assert.assertEquals(1, encodeCount.get());
scanner.close();
table.increment(b("z"), b("z"), 0L);
Assert.assertEquals(1, encodeCount.get());
table.commitTx();
table.postTxCommit();
// test that for the next tx, we encode again
tx = txSystemClient.startShort();
txRef.set(tx);
table.startTx(tx);
table.get(b("row"));
Assert.assertEquals(2, encodeCount.get());
table.commitTx();
// test that we encode again, even of postTxCommit was not called
tx = txSystemClient.startShort();
txRef.set(tx);
table.startTx(tx);
table.get(b("row"));
Assert.assertEquals(3, encodeCount.get());
table.commitTx();
table.rollbackTx();
// test that rollback does not encode the tx
Assert.assertEquals(3, encodeCount.get());
// test that we encode again if the previous tx rolled back
tx = txSystemClient.startShort();
txRef.set(tx);
table.startTx(tx);
table.get(b("row"));
Assert.assertEquals(4, encodeCount.get());
table.commitTx();
table.close();
Assert.assertEquals(4, encodeCount.get());
}
Aggregations