Examples with DatasetAdmin co.cask.cdap.api.dataset.DatasetAdmin used on opensource projects

Example 26 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class DatasetUpgrader method upgradeSystemDatasets.

private void upgradeSystemDatasets(ExecutorService executor) throws Exception {
    Map<String, Future<?>> futures = new HashMap<>();
    for (final DatasetSpecificationSummary spec : dsFramework.getInstances(NamespaceId.SYSTEM)) {
        final DatasetId datasetId = NamespaceId.SYSTEM.dataset(spec.getName());
        Runnable runnable = new Runnable() {

            public void run() {
                try {
                    LOG.info("Upgrading dataset in system namespace: {}, spec: {}", spec.getName(), spec.toString());
                    DatasetAdmin admin = dsFramework.getAdmin(datasetId, null);
                    // we know admin is not null, since we are looping over existing datasets
                    //noinspection ConstantConditions
                    admin.upgrade();
                    LOG.info("Upgraded dataset: {}", spec.getName());
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        };
        Future<?> future = executor.submit(runnable);
        futures.put(datasetId.toString(), future);
    }
    // Wait for the system dataset upgrades to complete
    Map<String, Throwable> failed = waitForUpgrade(futures);
    if (!failed.isEmpty()) {
        for (Map.Entry<String, Throwable> entry : failed.entrySet()) {
            LOG.error("Failed to upgrade system dataset {}", entry.getKey(), entry.getValue());
        }
        throw new Exception(String.format("Error upgrading system datasets. %s of %s failed", failed.size(), futures.size()));
    }
}

Example 27 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class PrefixedTable method register.

@Override
public void register(DatasetDefinitionRegistry registry) {
    DatasetDefinition<KeyValueTable, DatasetAdmin> kvTableDef = registry.get("keyValueTable");
    DatasetDefinition definition = new PrefixedTableDefinition("prefixedTable", kvTableDef);
    registry.add(definition);
}

Example 28 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class ObjectMappedTableDefinition method getDataset.

@Override
public ObjectMappedTableDataset<?> getDataset(DatasetContext datasetContext, DatasetSpecification spec, Map<String, String> arguments, ClassLoader classLoader) throws IOException {
    String keyName = ObjectMappedTableProperties.getRowKeyExploreName(spec.getProperties());
    DatasetSpecification tableSpec = spec.getSpecification(TABLE_NAME);
    // TODO: remove after CDAP-2122 is done
    if (!tableSpec.getProperties().containsKey(Table.PROPERTY_SCHEMA)) {
        tableSpec = DatasetSpecification.builder(tableSpec.getName(), tableSpec.getType()).properties(tableSpec.getProperties()).property(Table.PROPERTY_SCHEMA, spec.getProperty(Table.PROPERTY_SCHEMA)).property(Table.PROPERTY_SCHEMA_ROW_FIELD, keyName).datasets(tableSpec.getSpecifications().values()).build();
    }
    // reconstruct the table schema here because of backwards compatibility
    DatasetDefinition<Table, DatasetAdmin> tableDef = getDelegate(TABLE_NAME);
    Table table = tableDef.getDataset(datasetContext, tableSpec, arguments, classLoader);
    Map<String, String> properties = spec.getProperties();
    TypeRepresentation typeRep = GSON.fromJson(ObjectMappedTableProperties.getObjectTypeRepresentation(properties), TypeRepresentation.class);
    Schema objSchema = ObjectMappedTableProperties.getObjectSchema(properties);
    return new ObjectMappedTableDataset(spec.getName(), table, typeRep, objSchema, classLoader);
}

Example 29 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class TableTest method testClientSurvivesTableReset.

// this test ensures that an existing client survives the truncating or dropping and recreating of a table
@Test
public void testClientSurvivesTableReset() throws Exception {
    final String tableName = "survive";
    DatasetAdmin admin = getTableAdmin(CONTEXT1, tableName);
    admin.create();
    Table table = getTable(CONTEXT1, tableName);
    // write some values
    Transaction tx0 = txClient.startShort();
    ((TransactionAware) table).startTx(tx0);
    table.put(R1, a(C1), a(V1));
    Assert.assertTrue(txClient.canCommit(tx0, ((TransactionAware) table).getTxChanges()));
    Assert.assertTrue(((TransactionAware) table).commitTx());
    Assert.assertTrue(txClient.commit(tx0));
    ((TransactionAware) table).postTxCommit();
    // TableAssert.verify
    Transaction tx1 = txClient.startShort();
    ((TransactionAware) table).startTx(tx1);
    TableAssert.assertRow(a(C1, V1), table.get(R1));
    // drop table and recreate
    admin.drop();
    admin.create();
    // TableAssert.verify can read but nothing there
    TableAssert.assertRow(a(), table.get(R1));
    // only did read, safe to abort
    txClient.abort(tx1);
    // create a new client and write another value
    Table table2 = getTable(CONTEXT1, tableName);
    Transaction tx2 = txClient.startShort();
    ((TransactionAware) table2).startTx(tx2);
    table2.put(R1, a(C2), a(V2));
    Assert.assertTrue(txClient.canCommit(tx2, ((TransactionAware) table2).getTxChanges()));
    Assert.assertTrue(((TransactionAware) table2).commitTx());
    Assert.assertTrue(txClient.commit(tx2));
    ((TransactionAware) table2).postTxCommit();
    // TableAssert.verify it is visible
    Transaction tx3 = txClient.startShort();
    ((TransactionAware) table).startTx(tx3);
    TableAssert.assertRow(a(C2, V2), table.get(R1));
    // truncate table
    admin.truncate();
    // TableAssert.verify can read but nothing there
    TableAssert.assertRow(a(), table.get(R1));
    // only did read, safe to abort
    txClient.abort(tx3);
    // write again with other client
    Transaction tx4 = txClient.startShort();
    ((TransactionAware) table2).startTx(tx4);
    table2.put(R1, a(C3), a(V3));
    Assert.assertTrue(txClient.canCommit(tx4, ((TransactionAware) table2).getTxChanges()));
    Assert.assertTrue(((TransactionAware) table2).commitTx());
    Assert.assertTrue(txClient.commit(tx4));
    ((TransactionAware) table2).postTxCommit();
    // TableAssert.verify it is visible
    Transaction tx5 = txClient.startShort();
    ((TransactionAware) table).startTx(tx5);
    TableAssert.assertRow(a(C3, V3), table.get(R1));
    // only did read, safe to abort
    txClient.abort(tx5);
    // drop table
    admin.drop();
}

Example 30 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class BufferingTableTest method testMultiGetIncludesBuffer.

@Test
public void testMultiGetIncludesBuffer() throws Exception {
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try {
        // persist some data
        BufferingTable table = getTable(CONTEXT1, MY_TABLE);
        Transaction tx1 = txClient.startShort();
        table.startTx(tx1);
        // writing a couple rows
        // table should look like the following, with everything in the buffer
        //          c1    c2    c3    c4
        // r1       1     2     3     -
        // r2       -     3     2     1
        table.put(R1, a(C1, C2, C3), lb(1, 2, 3));
        table.put(R2, a(C2, C3, C4), lb(3, 2, 1));
        // check that multi-get can see buffered writes
        List<Row> rows = table.get(Lists.newArrayList(new Get(R1), new Get(R2)));
        Assert.assertEquals(2, rows.size());
        TableAssert.assertRow(rows.get(0), R1, a(C1, C2, C3), lb(1, 2, 3));
        TableAssert.assertRow(rows.get(1), R2, a(C2, C3, C4), lb(3, 2, 1));
        // check multi-get with gets that specify columns, and one get that should return an empty row
        rows = table.get(Lists.newArrayList(new Get(R1, C2, C3), new Get(R2, C2, C3), new Get(R3)));
        Assert.assertEquals(3, rows.size());
        TableAssert.assertRow(rows.get(0), R1, a(C2, C3), lb(2, 3));
        TableAssert.assertRow(rows.get(1), R2, a(C2, C3), lb(3, 2));
        Assert.assertTrue(rows.get(2).isEmpty());
        // persist changes
        Collection<byte[]> txChanges = table.getTxChanges();
        Assert.assertTrue(txClient.canCommit(tx1, txChanges));
        Assert.assertTrue(table.commitTx());
        Assert.assertTrue(txClient.commit(tx1));
        table.postTxCommit();
        // start another transaction
        Transaction tx2 = txClient.startShort();
        table.startTx(tx2);
        // now add another row, delete a row, and change some column values
        // table should look like the following
        //         c1    c2    c3    c4    c5
        // r1      -     -     3     2     -
        // r3      -     -     -     -     1
        table.put(R1, a(C2, C3, C4), lb(4, 3, 2));
        table.delete(R1, a(C1, C2));
        table.delete(R2);
        table.put(R3, C5, L1);
        // verify multi-get sees persisted data with buffer applied on top
        rows = table.get(Lists.newArrayList(new Get(R1), new Get(R2), new Get(R3)));
        Assert.assertEquals(3, rows.size());
        TableAssert.assertRow(rows.get(0), R1, a(C3, C4), lb(3, 2));
        Assert.assertTrue(rows.get(1).isEmpty());
        TableAssert.assertRow(rows.get(2), R3, a(C5), lb(1));
        // pretend there was a write conflict and rollback changes
        Assert.assertTrue(table.rollbackTx());
        txClient.abort(tx2);
        // start another transaction and make sure it can't see what was done before
        Transaction tx3 = txClient.startShort();
        table.startTx(tx3);
        rows = table.get(Lists.newArrayList(new Get(R1), new Get(R2)));
        Assert.assertEquals(2, rows.size());
        TableAssert.assertRow(rows.get(0), R1, a(C1, C2, C3), lb(1, 2, 3));
        TableAssert.assertRow(rows.get(1), R2, a(C2, C3, C4), lb(3, 2, 1));
    } finally {
        admin.drop();
    }
}