Examples with DatasetAdmin co.cask.cdap.api.dataset.DatasetAdmin used on opensource projects

Example 36 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class InMemoryDatasetOpExecutor method update.

@Override
public DatasetSpecification update(DatasetId datasetInstanceId, DatasetTypeMeta typeMeta, DatasetProperties props, DatasetSpecification existing) throws Exception {
    DatasetType type = client.getDatasetType(typeMeta, null, new ConstantClassLoaderProvider());
    if (type == null) {
        throw new IllegalArgumentException("Dataset type cannot be instantiated for provided type meta: " + typeMeta);
    }
    try {
        DatasetSpecification spec = type.reconfigure(datasetInstanceId.getEntityName(), props, existing);
        DatasetAdmin admin = type.getAdmin(DatasetContext.from(datasetInstanceId.getNamespace()), spec);
        if (admin instanceof Updatable) {
            ((Updatable) admin).update(existing);
        } else {
            admin.create();
        }
        if (spec.getDescription() == null && existing.getDescription() != null) {
            spec.setDescription(existing.getDescription());
        }
        return spec;
    } catch (IncompatibleUpdateException e) {
        throw new ConflictException(e.getMessage());
    }
}

Example 37 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testSimpleDataset.

@Test
public void testSimpleDataset() throws Exception {
    // Configuring Dataset types
    DatasetFramework framework = getFramework();
    // system namespace has a module orderedTable-inMemory
    Assert.assertTrue(framework.hasSystemType("table"));
    // myspace namespace has no modules
    Assert.assertFalse(framework.hasType(IN_MEMORY_TYPE));
    Assert.assertFalse(framework.hasType(SIMPLE_KV_TYPE));
    // add module to namespace 'myspace'
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    // make sure it got added to 'myspace'
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    // but not to 'system'
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    // Creating instance using a type from own namespace
    framework.addInstance(SimpleKVTable.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    // verify it got added to the right namespace
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    // and not to the system namespace
    Assert.assertFalse(framework.hasInstance(NamespaceId.SYSTEM.dataset("my_table")));
    // Doing some admin and data ops
    DatasetAdmin admin = framework.getAdmin(MY_TABLE, null);
    Assert.assertNotNull(admin);
    final SimpleKVTable kvTable = framework.getDataset(MY_TABLE, DatasetDefinition.NO_ARGUMENTS, null);
    Assert.assertNotNull(kvTable);
    TransactionExecutor txnl = new DefaultTransactionExecutor(new MinimalTxSystemClient(), (TransactionAware) kvTable);
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            kvTable.put("key1", "value1");
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals("value1", kvTable.get("key1"));
        }
    });
    admin.truncate();
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertTrue(kvTable.get("key1") == null);
        }
    });
    // cleanup
    framework.deleteInstance(MY_TABLE);
    framework.deleteModule(KEY_VALUE);
    // recreate instance without adding a module in 'myspace'. This should use types from default namespace
    framework.addInstance("table", MY_TABLE, DatasetProperties.EMPTY);
    // verify it got added to the right namespace
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    admin = framework.getAdmin(MY_TABLE, null);
    Assert.assertNotNull(admin);
    final Table table = framework.getDataset(MY_TABLE, DatasetDefinition.NO_ARGUMENTS, null);
    Assert.assertNotNull(table);
    txnl = new DefaultTransactionExecutor(new MinimalTxSystemClient(), (TransactionAware) table);
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            table.put(Bytes.toBytes("key1"), Bytes.toBytes("column1"), Bytes.toBytes("value1"));
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals("value1", Bytes.toString(table.get(Bytes.toBytes("key1"), Bytes.toBytes("column1"))));
        }
    });
    // cleanup
    framework.deleteInstance(MY_TABLE);
}

Example 38 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testCompositeDataset.

private void testCompositeDataset(DatasetFramework framework) throws Exception {
    // Doing some admin and data ops
    DatasetAdmin admin = framework.getAdmin(MY_TABLE, null);
    Assert.assertNotNull(admin);
    final KeyValueTable table = framework.getDataset(MY_TABLE, DatasetDefinition.NO_ARGUMENTS, null);
    Assert.assertNotNull(table);
    TransactionExecutor txnl = new DefaultTransactionExecutor(new MinimalTxSystemClient(), (TransactionAware) table);
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            table.put("key1", "value1");
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals("value1", table.get("key1"));
        }
    });
    admin.truncate();
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals(null, table.get("key1"));
        }
    });
}

Example 39 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class TableTest method testBasicCompareAndSwapWithTx.

@Test
public void testBasicCompareAndSwapWithTx() throws Exception {
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try {
        Transaction tx1 = txClient.startShort();
        Table myTable1 = getTable(CONTEXT1, MY_TABLE);
        ((TransactionAware) myTable1).startTx(tx1);
        // write r1->c1,v1 but not commit
        myTable1.put(R1, a(C1), a(V1));
        // write r1->c2,v2 but not commit
        Assert.assertTrue(myTable1.compareAndSwap(R1, C2, null, V5));
        // TableAssert.verify compare and swap result visible inside tx before commit
        TableAssert.assertRow(a(C1, V1, C2, V5), myTable1.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(V1, myTable1.get(R1, C1));
        Assert.assertArrayEquals(V5, myTable1.get(R1, C2));
        Assert.assertArrayEquals(null, myTable1.get(R1, C3));
        TableAssert.assertRow(a(C1, V1, C2, V5), myTable1.get(R1));
        // these should fail
        Assert.assertFalse(myTable1.compareAndSwap(R1, C1, null, V1));
        Assert.assertFalse(myTable1.compareAndSwap(R1, C1, V2, V1));
        Assert.assertFalse(myTable1.compareAndSwap(R1, C2, null, V2));
        Assert.assertFalse(myTable1.compareAndSwap(R1, C2, V2, V1));
        // but this should succeed
        Assert.assertTrue(myTable1.compareAndSwap(R1, C2, V5, V2));
        // start new tx (doesn't see changes of the tx1)
        Transaction tx2 = txClient.startShort();
        // committing tx1 in stages to check races are handled well
        // * first, flush operations of table
        Assert.assertTrue(txClient.canCommit(tx1, ((TransactionAware) myTable1).getTxChanges()));
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        // check that tx2 doesn't see changes (even though they were flushed) of tx1 by trying to compareAndSwap
        // assuming current value is null
        Table myTable2 = getTable(CONTEXT1, MY_TABLE);
        ((TransactionAware) myTable2).startTx(tx2);
        Assert.assertTrue(myTable2.compareAndSwap(R1, C1, null, V3));
        // start tx3 and TableAssert.verify same thing again
        Transaction tx3 = txClient.startShort();
        Table myTable3 = getTable(CONTEXT1, MY_TABLE);
        ((TransactionAware) myTable3).startTx(tx3);
        Assert.assertTrue(myTable3.compareAndSwap(R1, C1, null, V2));
        // * second, make tx visible
        Assert.assertTrue(txClient.commit(tx1));
        // TableAssert.verify that tx2 cannot commit because of the conflicts...
        Assert.assertFalse(txClient.canCommit(tx2, ((TransactionAware) myTable2).getTxChanges()));
        ((TransactionAware) myTable2).rollbackTx();
        txClient.abort(tx2);
        // start tx4 and TableAssert.verify that changes of tx1 are now visible
        Transaction tx4 = txClient.startShort();
        Table myTable4 = getTable(CONTEXT1, MY_TABLE);
        ((TransactionAware) myTable4).startTx(tx4);
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable4.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(V1, myTable4.get(R1, C1));
        Assert.assertArrayEquals(V2, myTable4.get(R1, C2));
        Assert.assertArrayEquals(null, myTable4.get(R1, C3));
        TableAssert.assertRow(a(C2, V2), myTable4.get(R1, a(C2)));
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable4.get(R1));
        // tx3 still cannot see tx1 changes
        Assert.assertTrue(myTable3.compareAndSwap(R1, C2, null, V5));
        // and it cannot commit because its changes cause conflicts
        Assert.assertFalse(txClient.canCommit(tx3, ((TransactionAware) myTable3).getTxChanges()));
        ((TransactionAware) myTable3).rollbackTx();
        txClient.abort(tx3);
        // TableAssert.verify we can do some ops with tx4 based on data written with tx1
        Assert.assertFalse(myTable4.compareAndSwap(R1, C1, null, V4));
        Assert.assertFalse(myTable4.compareAndSwap(R1, C2, null, V5));
        Assert.assertTrue(myTable4.compareAndSwap(R1, C1, V1, V3));
        Assert.assertTrue(myTable4.compareAndSwap(R1, C2, V2, V4));
        myTable4.delete(R1, a(C1));
        // committing tx4
        Assert.assertTrue(txClient.canCommit(tx4, ((TransactionAware) myTable3).getTxChanges()));
        Assert.assertTrue(((TransactionAware) myTable4).commitTx());
        Assert.assertTrue(txClient.commit(tx4));
        // TableAssert.verifying the result contents in next transaction
        Transaction tx5 = txClient.startShort();
        // NOTE: table instance can be re-used in series of transactions
        ((TransactionAware) myTable4).startTx(tx5);
        TableAssert.assertRow(a(C2, V4), myTable4.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(null, myTable4.get(R1, C1));
        Assert.assertArrayEquals(V4, myTable4.get(R1, C2));
        TableAssert.assertRow(a(C2, V4), myTable4.get(R1));
        Assert.assertTrue(txClient.canCommit(tx5, ((TransactionAware) myTable3).getTxChanges()));
        Assert.assertTrue(((TransactionAware) myTable3).commitTx());
        Assert.assertTrue(txClient.commit(tx5));
    } finally {
        admin.drop();
    }
}

Example 40 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class TableTest method testMetrics.

private void testMetrics(boolean readless) throws Exception {
    final String tableName = "survive";
    DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(readless).build();
    DatasetAdmin admin = getTableAdmin(CONTEXT1, tableName, props);
    admin.create();
    Table table = getTable(CONTEXT1, tableName, props);
    final Map<String, Long> metrics = Maps.newHashMap();
    ((MeteredDataset) table).setMetricsCollector(new MetricsCollector() {

        @Override
        public void increment(String metricName, long value) {
            Long old = metrics.get(metricName);
            metrics.put(metricName, old == null ? value : old + value);
        }

        @Override
        public void gauge(String metricName, long value) {
            metrics.put(metricName, value);
        }
    });
    // Note that we don't need to finish tx for metrics to be reported
    Transaction tx0 = txClient.startShort();
    ((TransactionAware) table).startTx(tx0);
    int writes = 0;
    int reads = 0;
    table.put(new Put(R1, C1, V1));
    verifyDatasetMetrics(metrics, ++writes, reads);
    table.compareAndSwap(R1, C1, V1, V2);
    verifyDatasetMetrics(metrics, ++writes, ++reads);
    // note: will not write anything as expected value will not match
    table.compareAndSwap(R1, C1, V1, V2);
    verifyDatasetMetrics(metrics, writes, ++reads);
    table.increment(new Increment(R2, C2, 1L));
    if (readless) {
        verifyDatasetMetrics(metrics, ++writes, reads);
    } else {
        verifyDatasetMetrics(metrics, ++writes, ++reads);
    }
    table.incrementAndGet(new Increment(R2, C2, 1L));
    verifyDatasetMetrics(metrics, ++writes, ++reads);
    table.get(new Get(R1, C1, V1));
    verifyDatasetMetrics(metrics, writes, ++reads);
    Scanner scanner = table.scan(new Scan(null, null));
    while (scanner.next() != null) {
        verifyDatasetMetrics(metrics, writes, ++reads);
    }
    table.delete(new Delete(R1, C1, V1));
    verifyDatasetMetrics(metrics, ++writes, reads);
    // drop table
    admin.drop();
}