Examples with DatasetAdmin io.cdap.cdap.api.dataset.DatasetAdmin used on opensource projects

Example 76 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class NoSqlStructuredTableRegistry method initIfNeeded.

private void initIfNeeded() throws IOException {
    if (initialized) {
        return;
    }
    synchronized (this) {
        if (initialized) {
            return;
        }
        DatasetAdmin admin = tableDefinition.getAdmin(SYSTEM_CONTEXT, entityRegistrySpec, null);
        if (!admin.exists()) {
            LOG.info("Creating dataset table {} in namespace {}", entityRegistrySpec.getName(), NamespaceId.SYSTEM);
            admin.create();
        }
        initialized = true;
    }
}

Example 77 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class DatasetUpgrader method upgradeSystemDatasets.

private void upgradeSystemDatasets(ExecutorService executor) throws Exception {
    Map<String, Future<?>> futures = new HashMap<>();
    for (final DatasetSpecificationSummary spec : dsFramework.getInstances(NamespaceId.SYSTEM)) {
        final DatasetId datasetId = NamespaceId.SYSTEM.dataset(spec.getName());
        Runnable runnable = new Runnable() {

            @Override
            public void run() {
                try {
                    LOG.info("Upgrading dataset in system namespace: {}, spec: {}", spec.getName(), spec.toString());
                    DatasetAdmin admin = dsFramework.getAdmin(datasetId, null);
                    // we know admin is not null, since we are looping over existing datasets
                    // noinspection ConstantConditions
                    admin.upgrade();
                    LOG.info("Upgraded dataset: {}", spec.getName());
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        };
        Future<?> future = executor.submit(runnable);
        futures.put(datasetId.toString(), future);
    }
    // Wait for the system dataset upgrades to complete
    Map<String, Throwable> failed = waitForUpgrade(futures);
    if (!failed.isEmpty()) {
        for (Map.Entry<String, Throwable> entry : failed.entrySet()) {
            LOG.error("Failed to upgrade system dataset {}", entry.getKey(), entry.getValue());
        }
        throw new Exception(String.format("Error upgrading system datasets. %s of %s failed", failed.size(), futures.size()));
    }
}

Example 78 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class DatasetUpgrader method upgradeUserTable.

private void upgradeUserTable(HTableDescriptor desc) throws IOException {
    TableId tableId = HTableNameConverter.from(desc);
    LOG.info("Upgrading hbase table: {}, desc: {}", tableId, desc);
    final boolean supportsIncrement = HBaseTableAdmin.supportsReadlessIncrements(desc);
    final boolean transactional = HBaseTableAdmin.isTransactional(desc);
    DatasetAdmin admin = new AbstractHBaseDataSetAdmin(tableId, hConf, cConf, hBaseTableUtil, locationFactory) {

        @Override
        protected CoprocessorJar createCoprocessorJar() throws IOException {
            return HBaseTableAdmin.createCoprocessorJarInternal(cConf, coprocessorManager, hBaseTableUtil, transactional, supportsIncrement);
        }

        @Override
        protected boolean needsUpdate(HTableDescriptor tableDescriptor, HTableDescriptorBuilder descriptorBuilder) {
            return false;
        }

        @Override
        public void create() throws IOException {
            // no-op
            throw new UnsupportedOperationException("This DatasetAdmin is only used for upgrade() operation");
        }
    };
    admin.upgrade();
    LOG.info("Upgraded hbase table: {}", tableId);
}

Example 79 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by cdapio.

the class TableTest method testBasicDeleteWithTx.

@Test
public void testBasicDeleteWithTx() throws Exception {
    // we will test 3 different delete column ops and one delete row op
    // * delete column with delete
    // * delete column with put null value
    // * delete column with put byte[0] value
    // * delete row with delete
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try (Table myTable1 = getTable(CONTEXT1, MY_TABLE);
        Table myTable2 = getTable(CONTEXT1, MY_TABLE)) {
        // write smth and commit
        Transaction tx1 = txClient.startShort();
        ((TransactionAware) myTable1).startTx(tx1);
        myTable1.put(R1, a(C1, C2), a(V1, V2));
        myTable1.put(R2, a(C1, C2), a(V2, V3));
        myTable1.put(R3, a(C1, C2), a(V3, V4));
        myTable1.put(R4, a(C1, C2), a(V4, V5));
        txClient.canCommitOrThrow(tx1, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx1);
        // Now, we will test delete ops
        // start new tx2
        Transaction tx2 = txClient.startShort();
        ((TransactionAware) myTable2).startTx(tx2);
        // TableAssert.verify tx2 sees changes of tx1
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable2.get(R1, a(C1, C2)));
        // TableAssert.verify tx2 sees changes of tx1
        TableAssert.assertRow(a(C1, V2, C2, V3), myTable2.get(R2));
        // delete c1, r2
        myTable2.delete(R1, a(C1));
        myTable2.delete(R2);
        // same as delete a column
        myTable2.put(R3, C1, null);
        // TableAssert.verify can see deletes in own changes before commit
        TableAssert.assertRow(a(C2, V2), myTable2.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(null, myTable2.get(R1, C1));
        Assert.assertArrayEquals(V2, myTable2.get(R1, C2));
        TableAssert.assertRow(a(), myTable2.get(R2));
        TableAssert.assertRow(a(C2, V4), myTable2.get(R3));
        // overwrite c2 and write new value to c1
        myTable2.put(R1, a(C1, C2), a(V3, V4));
        myTable2.put(R2, a(C1, C2), a(V4, V5));
        myTable2.put(R3, a(C1, C2), a(V1, V2));
        myTable2.put(R4, a(C1, C2), a(V2, V3));
        // TableAssert.verify can see changes in own changes before commit
        TableAssert.assertRow(a(C1, V3, C2, V4), myTable2.get(R1, a(C1, C2, C3)));
        Assert.assertArrayEquals(V3, myTable2.get(R1, C1));
        Assert.assertArrayEquals(V4, myTable2.get(R1, C2));
        Assert.assertArrayEquals(null, myTable2.get(R1, C3));
        TableAssert.assertRow(a(C1, V4, C2, V5), myTable2.get(R2));
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable2.get(R3));
        TableAssert.assertRow(a(C1, V2, C2, V3), myTable2.get(R4));
        // delete c2 and r2
        myTable2.delete(R1, a(C2));
        myTable2.delete(R2);
        myTable2.put(R1, C2, null);
        // TableAssert.verify that delete is there (i.e. not reverted to whatever was persisted before)
        TableAssert.assertRow(a(C1, V3), myTable2.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(V3, myTable2.get(R1, C1));
        Assert.assertArrayEquals(null, myTable2.get(R1, C2));
        TableAssert.assertRow(a(), myTable2.get(R2));
        Assert.assertArrayEquals(V1, myTable2.get(R3, C1));
        Assert.assertArrayEquals(V2, myTable2.get(R4, C1));
        // start tx3 and TableAssert.verify that changes of tx2 are not visible yet
        Transaction tx3 = txClient.startShort();
        // NOTE: table instance can be re-used between tx
        ((TransactionAware) myTable1).startTx(tx3);
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(V1, myTable1.get(R1, C1));
        Assert.assertArrayEquals(V2, myTable1.get(R1, C2));
        Assert.assertArrayEquals(null, myTable1.get(R1, C3));
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1));
        TableAssert.assertRow(a(C1, V2, C2, V3), myTable1.get(R2));
        TableAssert.assertRow(a(C1, V3, C2, V4), myTable1.get(R3));
        TableAssert.assertRow(a(C1, V4, C2, V5), myTable1.get(R4));
        txClient.canCommitOrThrow(tx3, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx3);
        // starting tx4 before committing tx2 so that we can check conflicts are detected wrt deletes
        Transaction tx4 = txClient.startShort();
        // starting tx5 before committing tx2 so that we can check conflicts are detected wrt deletes
        Transaction tx5 = txClient.startShort();
        // commit tx2 in stages to see how races are handled wrt delete ops
        txClient.canCommitOrThrow(tx2, ((TransactionAware) myTable2).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable2).commitTx());
        // start tx6 and TableAssert.verify that changes of tx2 are not visible yet (even though they are flushed)
        Transaction tx6 = txClient.startShort();
        // NOTE: table instance can be re-used between tx
        ((TransactionAware) myTable1).startTx(tx6);
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(V1, myTable1.get(R1, C1));
        Assert.assertArrayEquals(V2, myTable1.get(R1, C2));
        Assert.assertArrayEquals(null, myTable1.get(R1, C3));
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1));
        TableAssert.assertRow(a(C1, V2, C2, V3), myTable1.get(R2));
        TableAssert.assertRow(a(C1, V3, C2, V4), myTable1.get(R3));
        TableAssert.assertRow(a(C1, V4, C2, V5), myTable1.get(R4));
        txClient.canCommitOrThrow(tx6, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx6);
        // make tx2 visible
        txClient.commitOrThrow(tx2);
        // start tx7 and TableAssert.verify that changes of tx2 are now visible
        Transaction tx7 = txClient.startShort();
        // NOTE: table instance can be re-used between tx
        ((TransactionAware) myTable1).startTx(tx7);
        TableAssert.assertRow(a(C1, V3), myTable1.get(R1, a(C1, C2)));
        TableAssert.assertRow(a(C1, V3), myTable1.get(R1));
        Assert.assertArrayEquals(V3, myTable1.get(R1, C1));
        Assert.assertArrayEquals(null, myTable1.get(R1, C2));
        TableAssert.assertRow(a(C1, V3), myTable1.get(R1, a(C1, C2)));
        TableAssert.assertRow(a(), myTable1.get(R2));
        Assert.assertArrayEquals(V1, myTable1.get(R3, C1));
        Assert.assertArrayEquals(V2, myTable1.get(R4, C1));
        txClient.canCommitOrThrow(tx6, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx7);
        // but not visible to tx4 that we started earlier than tx2 became visible
        // NOTE: table instance can be re-used between tx
        ((TransactionAware) myTable1).startTx(tx4);
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(V1, myTable1.get(R1, C1));
        Assert.assertArrayEquals(V2, myTable1.get(R1, C2));
        Assert.assertArrayEquals(null, myTable1.get(R1, C3));
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1));
        TableAssert.assertRow(a(C1, V2, C2, V3), myTable1.get(R2));
        TableAssert.assertRow(a(C1, V3, C2, V4), myTable1.get(R3));
        TableAssert.assertRow(a(C1, V4, C2, V5), myTable1.get(R4));
        // writing to deleted column, to check conflicts are detected (delete-write conflict)
        myTable1.put(R1, a(C2), a(V5));
        try {
            txClient.canCommitOrThrow(tx4, ((TransactionAware) myTable1).getTxChanges());
            Assert.fail("Conflict not detected!");
        } catch (TransactionConflictException e) {
        // expected
        }
        ((TransactionAware) myTable1).rollbackTx();
        txClient.abort(tx4);
        // deleting changed column, to check conflicts are detected (write-delete conflict)
        // NOTE: table instance can be re-used between tx
        ((TransactionAware) myTable1).startTx(tx5);
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(V1, myTable1.get(R1, C1));
        Assert.assertArrayEquals(V2, myTable1.get(R1, C2));
        Assert.assertArrayEquals(null, myTable1.get(R1, C3));
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1));
        TableAssert.assertRow(a(C1, V2, C2, V3), myTable1.get(R2));
        TableAssert.assertRow(a(C1, V3, C2, V4), myTable1.get(R3));
        TableAssert.assertRow(a(C1, V4, C2, V5), myTable1.get(R4));
        // NOTE: we are TableAssert.verifying conflict in one operation only. We may want to test each...
        myTable1.delete(R1, a(C1));
        try {
            txClient.canCommitOrThrow(tx5, ((TransactionAware) myTable1).getTxChanges());
            Assert.fail("Conflict not detected!");
        } catch (TransactionConflictException e) {
        // expected
        }
        ((TransactionAware) myTable1).rollbackTx();
        txClient.abort(tx5);
    } finally {
        admin.drop();
    }
}

Example 80 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by cdapio.

the class TableTest method testMultiGetWithEmpty.

@Test
public void testMultiGetWithEmpty() throws Exception {
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try (Table myTable = getTable(CONTEXT1, MY_TABLE)) {
        Transaction tx = txClient.startShort();
        ((TransactionAware) myTable).startTx(tx);
        myTable.put(R1, C1, V1);
        myTable.put(R1, C2, V2);
        myTable.put(R1, C3, V3);
        myTable.put(R1, C4, V4);
        List<Get> gets = new ArrayList<>();
        // the second and fourth Gets are requesting 0 columns. This tests correctness of batch-get logic, when there
        // is/are empty Gets among them.
        gets.add(new Get(R1, C1));
        gets.add(new Get(R1, ImmutableList.<byte[]>of()));
        gets.add(new Get(R1, C2, C3));
        gets.add(new Get(R1, ImmutableList.<byte[]>of()));
        gets.add(new Get(R1, C4));
        List<Row> rows = myTable.get(gets);
        // first off, the Gets at index two and four should be empty
        Assert.assertEquals(0, rows.get(1).getColumns().size());
        Assert.assertEquals(0, rows.get(3).getColumns().size());
        // verify the results of the other Gets
        Assert.assertEquals(1, rows.get(0).getColumns().size());
        Assert.assertArrayEquals(V1, rows.get(0).get(C1));
        Assert.assertEquals(2, rows.get(2).getColumns().size());
        Assert.assertArrayEquals(V2, rows.get(2).get(C2));
        Assert.assertArrayEquals(V3, rows.get(2).get(C3));
        Assert.assertEquals(1, rows.get(4).getColumns().size());
        Assert.assertArrayEquals(V4, rows.get(4).get(C4));
        txClient.abort(tx);
    } finally {
        admin.drop();
    }
}