use of io.cdap.cdap.api.dataset.table.Put in project cdap by caskdata.
the class NodeStatesAction method run.
@Override
public void run(BatchActionContext context) throws Exception {
Table table = context.getDataset(conf.tableName);
for (Map.Entry<String, WorkflowNodeState> entry : context.getNodeStates().entrySet()) {
Put put = new Put(entry.getKey());
WorkflowNodeState nodeState = entry.getValue();
put.add("runid", nodeState.getRunId());
put.add("nodeid", nodeState.getNodeId());
put.add("status", nodeState.getNodeStatus().name());
table.put(put);
}
}
use of io.cdap.cdap.api.dataset.table.Put in project cdap by caskdata.
the class CharCountProgram method initialize.
@Override
public void initialize() throws Exception {
SparkClientContext context = getContext();
context.setSparkConf(new SparkConf().set("spark.io.compression.codec", "org.apache.spark.io.LZFCompressionCodec"));
Table totals = context.getDataset("totals");
totals.get(new Get("total").add("total")).getLong("total");
totals.put(new Put("total").add("total", 0L));
}
use of io.cdap.cdap.api.dataset.table.Put in project cdap by caskdata.
the class AdminAppTestRun method testAdminService.
@Test
public void testAdminService() throws Exception {
// Start the service
ServiceManager serviceManager = appManager.getServiceManager(AdminApp.SERVICE_NAME).start();
String namespaceX = "x";
try {
URI serviceURI = serviceManager.getServiceURL(10, TimeUnit.SECONDS).toURI();
// dataset nn should not exist
HttpResponse response = executeHttp(HttpRequest.get(serviceURI.resolve("exists/nn").toURL()).build());
Assert.assertEquals(200, response.getResponseCode());
Assert.assertEquals("false", response.getResponseBodyAsString());
// create nn as a table
response = executeHttp(HttpRequest.put(serviceURI.resolve("create/nn/table").toURL()).build());
Assert.assertEquals(200, response.getResponseCode());
// now nn should exist
response = executeHttp(HttpRequest.get(serviceURI.resolve("exists/nn").toURL()).build());
Assert.assertEquals(200, response.getResponseCode());
Assert.assertEquals("true", response.getResponseBodyAsString());
// create it again as a fileset -> should fail with conflict
response = executeHttp(HttpRequest.put(serviceURI.resolve("create/nn/fileSet").toURL()).build());
Assert.assertEquals(409, response.getResponseCode());
// get the type for xx -> not found
response = executeHttp(HttpRequest.get(serviceURI.resolve("type/xx").toURL()).build());
Assert.assertEquals(404, response.getResponseCode());
// get the type for nn -> table
response = executeHttp(HttpRequest.get(serviceURI.resolve("type/nn").toURL()).build());
Assert.assertEquals(200, response.getResponseCode());
Assert.assertEquals("table", response.getResponseBodyAsString());
// update xx's properties -> should get not-found
Map<String, String> nnProps = TableProperties.builder().setTTL(1000L).build().getProperties();
response = executeHttp(HttpRequest.put(serviceURI.resolve("update/xx").toURL()).withBody(GSON.toJson(nnProps)).build());
Assert.assertEquals(404, response.getResponseCode());
// update nn's properties
response = executeHttp(HttpRequest.put(serviceURI.resolve("update/nn").toURL()).withBody(GSON.toJson(nnProps)).build());
Assert.assertEquals(200, response.getResponseCode());
// get properties for xx -> not found
response = executeHttp(HttpRequest.get(serviceURI.resolve("props/xx").toURL()).build());
Assert.assertEquals(404, response.getResponseCode());
// get properties for nn and validate
response = executeHttp(HttpRequest.get(serviceURI.resolve("props/nn").toURL()).build());
Assert.assertEquals(200, response.getResponseCode());
Map<String, String> returnedProps = GSON.fromJson(response.getResponseBodyAsString(), new TypeToken<Map<String, String>>() {
}.getType());
Assert.assertEquals(nnProps, returnedProps);
// write some data to the table
DataSetManager<Table> nnManager = getDataset("nn");
nnManager.get().put(new Put("x", "y", "z"));
nnManager.flush();
// in a new tx, validate that data is in table
Assert.assertFalse(nnManager.get().get(new Get("x")).isEmpty());
Assert.assertEquals("z", nnManager.get().get(new Get("x", "y")).getString("y"));
nnManager.flush();
// truncate xx -> not found
response = executeHttp(HttpRequest.post(serviceURI.resolve("truncate/xx").toURL()).build());
Assert.assertEquals(404, response.getResponseCode());
// truncate nn
response = executeHttp(HttpRequest.post(serviceURI.resolve("truncate/nn").toURL()).build());
Assert.assertEquals(200, response.getResponseCode());
// validate table is empty
Assert.assertTrue(nnManager.get().get(new Get("x")).isEmpty());
nnManager.flush();
// delete nn
response = executeHttp(HttpRequest.delete(serviceURI.resolve("delete/nn").toURL()).build());
Assert.assertEquals(200, response.getResponseCode());
// delete again -> not found
response = executeHttp(HttpRequest.delete(serviceURI.resolve("delete/nn").toURL()).build());
Assert.assertEquals(404, response.getResponseCode());
// delete xx which never existed -> not found
response = executeHttp(HttpRequest.delete(serviceURI.resolve("delete/xx").toURL()).build());
Assert.assertEquals(404, response.getResponseCode());
// exists should now return false for nn
response = executeHttp(HttpRequest.get(serviceURI.resolve("exists/nn").toURL()).build());
Assert.assertEquals(200, response.getResponseCode());
Assert.assertEquals("false", response.getResponseBodyAsString());
Assert.assertNull(getDataset("nn").get());
// test Admin.namespaceExists()
HttpRequest request = HttpRequest.get(serviceURI.resolve("namespaces/y").toURL()).build();
response = executeHttp(request);
Assert.assertEquals(404, response.getResponseCode());
// test Admin.getNamespaceSummary()
NamespaceMeta namespaceXMeta = new NamespaceMeta.Builder().setName(namespaceX).setGeneration(10L).build();
getNamespaceAdmin().create(namespaceXMeta);
request = HttpRequest.get(serviceURI.resolve("namespaces/" + namespaceX).toURL()).build();
response = executeHttp(request);
NamespaceSummary namespaceSummary = GSON.fromJson(response.getResponseBodyAsString(), NamespaceSummary.class);
NamespaceSummary expectedX = new NamespaceSummary(namespaceXMeta.getName(), namespaceXMeta.getDescription(), namespaceXMeta.getGeneration());
Assert.assertEquals(expectedX, namespaceSummary);
// test ArtifactManager.listArtifacts()
ArtifactId pluginArtifactId = new NamespaceId(namespaceX).artifact("r1", "1.0.0");
// add a plugin artifact to namespace X
addPluginArtifact(pluginArtifactId, ADMIN_APP_ARTIFACT, DummyPlugin.class);
// no plugins should be listed in the default namespace, but the app artifact should
request = HttpRequest.get(serviceURI.resolve("namespaces/default/plugins").toURL()).build();
response = executeHttp(request);
Assert.assertEquals(200, response.getResponseCode());
Type setType = new TypeToken<Set<ArtifactSummary>>() {
}.getType();
Assert.assertEquals(Collections.singleton(ADMIN_ARTIFACT_SUMMARY), GSON.fromJson(response.getResponseBodyAsString(), setType));
// the plugin should be listed in namespace X
request = HttpRequest.get(serviceURI.resolve("namespaces/x/plugins").toURL()).build();
response = executeHttp(request);
Assert.assertEquals(200, response.getResponseCode());
ArtifactSummary expected = new ArtifactSummary(pluginArtifactId.getArtifact(), pluginArtifactId.getVersion());
Assert.assertEquals(Collections.singleton(expected), GSON.fromJson(response.getResponseBodyAsString(), setType));
} finally {
serviceManager.stop();
if (getNamespaceAdmin().exists(new NamespaceId(namespaceX))) {
getNamespaceAdmin().delete(new NamespaceId(namespaceX));
}
}
}
use of io.cdap.cdap.api.dataset.table.Put in project cdap by caskdata.
the class HBaseTableTest method testScannerCache.
@Test
public void testScannerCache() throws Exception {
String tableName = "scanCache";
// note: it appears that HBase only enforces the scanner timeout after 10 seconds.
// setting it to 3 seconds does not mean it will actually fsail after 3 sweconds.
// therefore we have to cross the 10 seconds. here: 1200 times 10ms sleep.
int numRows = 1200;
DatasetAdmin admin = getTableAdmin(CONTEXT1, tableName);
admin.create();
try (Table myTable1 = getTable(CONTEXT1, tableName)) {
// write some rows and commit
Transaction tx1 = txClient.startShort();
((TransactionAware) myTable1).startTx(tx1);
for (int i = 0; i < numRows; i++) {
myTable1.put(new Put("" + i, "x", "y"));
}
txClient.canCommitOrThrow(tx1, ((TransactionAware) myTable1).getTxChanges());
Assert.assertTrue(((TransactionAware) myTable1).commitTx());
txClient.commitOrThrow(tx1);
try {
testScannerCache(numRows, tableName, null, null, null);
Assert.fail("this should have failed with ScannerTimeoutException");
} catch (Exception e) {
// we expect a RuntimeException wrapping an HBase ScannerTimeoutException
if (!(e.getCause() instanceof ScannerTimeoutException)) {
throw e;
}
}
// cache=100 as dataset property
testScannerCache(numRows, tableName, "100", null, null);
// cache=100 as dataset runtime argument
testScannerCache(numRows, tableName, "1000", "100", null);
// cache=100 as scan property
testScannerCache(numRows, tableName, "5000", "1000", "100");
} finally {
admin.drop();
}
}
use of io.cdap.cdap.api.dataset.table.Put in project cdap by caskdata.
the class HBaseTableTest method testColumnFamily.
@Test
public void testColumnFamily() throws Exception {
DatasetProperties props = TableProperties.builder().setColumnFamily("t").build();
String tableName = "testcf";
DatasetAdmin admin = getTableAdmin(CONTEXT1, tableName, props);
admin.create();
try (BufferingTable table = getTable(CONTEXT1, tableName, props);
BufferingTable table2 = getTable(CONTEXT1, tableName, props)) {
TransactionSystemClient txClient = new DetachedTxSystemClient();
TransactionExecutor executor = new DefaultTransactionExecutor(txClient, table);
executor.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
table.put(new Put("row", "column", "testValue"));
}
});
executor = new DefaultTransactionExecutor(txClient, table2);
executor.execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
Assert.assertEquals("testValue", table2.get(new Get("row", "column")).getString("column"));
}
});
// Verify the column family name
TableId hTableId = hBaseTableUtil.createHTableId(new NamespaceId(CONTEXT1.getNamespaceId()), tableName);
HTableDescriptor htd = hBaseTableUtil.getHTableDescriptor(TEST_HBASE.getHBaseAdmin(), hTableId);
HColumnDescriptor hcd = htd.getFamily(Bytes.toBytes("t"));
Assert.assertNotNull(hcd);
Assert.assertEquals("t", hcd.getNameAsString());
}
}
Aggregations