Examples with DatasetAdmin co.cask.cdap.api.dataset.DatasetAdmin used on opensource projects

Example 46 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class BufferingTableTest method testMultiGetIncludesBuffer.

@Test
public void testMultiGetIncludesBuffer() throws Exception {
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try {
        // persist some data
        BufferingTable table = getTable(CONTEXT1, MY_TABLE);
        Transaction tx1 = txClient.startShort();
        table.startTx(tx1);
        // writing a couple rows
        // table should look like the following, with everything in the buffer
        //          c1    c2    c3    c4
        // r1       1     2     3     -
        // r2       -     3     2     1
        table.put(R1, a(C1, C2, C3), lb(1, 2, 3));
        table.put(R2, a(C2, C3, C4), lb(3, 2, 1));
        // check that multi-get can see buffered writes
        List<Row> rows = table.get(Lists.newArrayList(new Get(R1), new Get(R2)));
        Assert.assertEquals(2, rows.size());
        TableAssert.assertRow(rows.get(0), R1, a(C1, C2, C3), lb(1, 2, 3));
        TableAssert.assertRow(rows.get(1), R2, a(C2, C3, C4), lb(3, 2, 1));
        // check multi-get with gets that specify columns, and one get that should return an empty row
        rows = table.get(Lists.newArrayList(new Get(R1, C2, C3), new Get(R2, C2, C3), new Get(R3)));
        Assert.assertEquals(3, rows.size());
        TableAssert.assertRow(rows.get(0), R1, a(C2, C3), lb(2, 3));
        TableAssert.assertRow(rows.get(1), R2, a(C2, C3), lb(3, 2));
        Assert.assertTrue(rows.get(2).isEmpty());
        // persist changes
        Collection<byte[]> txChanges = table.getTxChanges();
        Assert.assertTrue(txClient.canCommit(tx1, txChanges));
        Assert.assertTrue(table.commitTx());
        Assert.assertTrue(txClient.commit(tx1));
        table.postTxCommit();
        // start another transaction
        Transaction tx2 = txClient.startShort();
        table.startTx(tx2);
        // now add another row, delete a row, and change some column values
        // table should look like the following
        //         c1    c2    c3    c4    c5
        // r1      -     -     3     2     -
        // r3      -     -     -     -     1
        table.put(R1, a(C2, C3, C4), lb(4, 3, 2));
        table.delete(R1, a(C1, C2));
        table.delete(R2);
        table.put(R3, C5, L1);
        // verify multi-get sees persisted data with buffer applied on top
        rows = table.get(Lists.newArrayList(new Get(R1), new Get(R2), new Get(R3)));
        Assert.assertEquals(3, rows.size());
        TableAssert.assertRow(rows.get(0), R1, a(C3, C4), lb(3, 2));
        Assert.assertTrue(rows.get(1).isEmpty());
        TableAssert.assertRow(rows.get(2), R3, a(C5), lb(1));
        // pretend there was a write conflict and rollback changes
        Assert.assertTrue(table.rollbackTx());
        txClient.abort(tx2);
        // start another transaction and make sure it can't see what was done before
        Transaction tx3 = txClient.startShort();
        table.startTx(tx3);
        rows = table.get(Lists.newArrayList(new Get(R1), new Get(R2)));
        Assert.assertEquals(2, rows.size());
        TableAssert.assertRow(rows.get(0), R1, a(C1, C2, C3), lb(1, 2, 3));
        TableAssert.assertRow(rows.get(1), R2, a(C2, C3, C4), lb(3, 2, 1));
    } finally {
        admin.drop();
    }
}

Example 47 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class BufferingTableTest method testTxChangePrefix.

// This test is in Buffering table because it needs to test the transaction change prefix
@Test
public void testTxChangePrefix() throws Exception {
    String tableName = "same";
    DatasetAdmin admin1 = getTableAdmin(CONTEXT1, tableName);
    DatasetAdmin admin2 = getTableAdmin(CONTEXT2, tableName);
    admin1.create();
    admin2.create();
    try {
        BufferingTable table1 = getTable(CONTEXT1, tableName);
        BufferingTable table2 = getTable(CONTEXT2, tableName);
        // write some values in table1
        Transaction tx1 = txClient.startShort();
        table1.startTx(tx1);
        table1.put(R1, a(C1), a(V1));
        Collection<byte[]> tx1Changes = table1.getTxChanges();
        Assert.assertTrue(txClient.canCommit(tx1, tx1Changes));
        Assert.assertTrue(table1.commitTx());
        Assert.assertTrue(txClient.commit(tx1));
        table1.postTxCommit();
        // write some values in table2
        Transaction tx2 = txClient.startShort();
        table2.startTx(tx2);
        table2.put(R1, a(C1), a(V1));
        Collection<byte[]> tx2Changes = table2.getTxChanges();
        Assert.assertTrue(txClient.canCommit(tx2, tx2Changes));
        Assert.assertTrue(table2.commitTx());
        Assert.assertTrue(txClient.commit(tx2));
        table1.postTxCommit();
        String tx1ChangePrefix = new String(table1.getNameAsTxChangePrefix());
        String tx2ChangePrefix = new String(table2.getNameAsTxChangePrefix());
        String tx1Change = new String(((ArrayList<byte[]>) tx1Changes).get(0));
        String tx2Change = new String(((ArrayList<byte[]>) tx2Changes).get(0));
        Assert.assertNotEquals(tx1ChangePrefix, tx2ChangePrefix);
        Assert.assertTrue(tx1ChangePrefix.contains(NAMESPACE1.getEntityName()));
        Assert.assertTrue(tx2ChangePrefix.contains(NAMESPACE2.getEntityName()));
        Assert.assertTrue(tx1Change.startsWith(tx1ChangePrefix));
        Assert.assertTrue(tx2Change.startsWith(tx2ChangePrefix));
    } finally {
        admin1.drop();
        admin2.drop();
    }
}

Example 48 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class PrefixedTable method register.

@Override
public void register(DatasetDefinitionRegistry registry) {
    DatasetDefinition<KeyValueTable, DatasetAdmin> kvTableDef = registry.get("keyValueTable");
    DatasetDefinition definition = new PrefixedTableDefinition("prefixedTable", kvTableDef);
    registry.add(definition);
}

Example 49 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class DatasetUpgrader method upgradeUserTable.

private void upgradeUserTable(HTableDescriptor desc) throws IOException {
    TableId tableId = HTableNameConverter.from(desc);
    LOG.info("Upgrading hbase table: {}, desc: {}", tableId, desc);
    final boolean supportsIncrement = HBaseTableAdmin.supportsReadlessIncrements(desc);
    final boolean transactional = HBaseTableAdmin.isTransactional(desc);
    DatasetAdmin admin = new AbstractHBaseDataSetAdmin(tableId, hConf, cConf, hBaseTableUtil, locationFactory) {

        @Override
        protected CoprocessorJar createCoprocessorJar() throws IOException {
            return HBaseTableAdmin.createCoprocessorJarInternal(cConf, coprocessorManager, hBaseTableUtil, transactional, supportsIncrement);
        }

        @Override
        protected boolean needsUpdate(HTableDescriptor tableDescriptor) {
            return false;
        }

        @Override
        public void create() throws IOException {
            // no-op
            throw new UnsupportedOperationException("This DatasetAdmin is only used for upgrade() operation");
        }
    };
    admin.upgrade();
    LOG.info("Upgraded hbase table: {}", tableId);
}

Example 50 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class DatasetUpgrader method upgradeSystemDatasets.

private void upgradeSystemDatasets(ExecutorService executor) throws Exception {
    Map<String, Future<?>> futures = new HashMap<>();
    for (final DatasetSpecificationSummary spec : dsFramework.getInstances(NamespaceId.SYSTEM)) {
        final DatasetId datasetId = NamespaceId.SYSTEM.dataset(spec.getName());
        Runnable runnable = new Runnable() {

            public void run() {
                try {
                    LOG.info("Upgrading dataset in system namespace: {}, spec: {}", spec.getName(), spec.toString());
                    DatasetAdmin admin = dsFramework.getAdmin(datasetId, null);
                    // we know admin is not null, since we are looping over existing datasets
                    //noinspection ConstantConditions
                    admin.upgrade();
                    LOG.info("Upgraded dataset: {}", spec.getName());
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        };
        Future<?> future = executor.submit(runnable);
        futures.put(datasetId.toString(), future);
    }
    // Wait for the system dataset upgrades to complete
    Map<String, Throwable> failed = waitForUpgrade(futures);
    if (!failed.isEmpty()) {
        for (Map.Entry<String, Throwable> entry : failed.entrySet()) {
            LOG.error("Failed to upgrade system dataset {}", entry.getKey(), entry.getValue());
        }
        throw new Exception(String.format("Error upgrading system datasets. %s of %s failed", failed.size(), futures.size()));
    }
}