Examples with DatasetAdmin io.cdap.cdap.api.dataset.DatasetAdmin used on opensource projects

Example 16 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class TableTest method testStringPutGet.

/**
 * Test that tables convert String to byte[] and back inversely. That is, get(put(s)) == s.
 */
@Test
public void testStringPutGet() throws Exception {
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try (Table t = getTable(CONTEXT1, MY_TABLE)) {
        // ascii
        testStringPutGet(t, "a", "b", "c");
        // row is latin
        testStringPutGet(t, "ä", "b", "c");
        // column is latin
        testStringPutGet(t, "a", "ä", "c");
        // value is latin
        testStringPutGet(t, "a", "b", "ä");
        // row is non-printable
        testStringPutGet(t, "\u009F", "b", "c");
        // column is non-printable
        testStringPutGet(t, "a", "\u009F", "c");
        // value is non-printable
        testStringPutGet(t, "a", "b", "\u009F");
    } finally {
        admin.drop();
    }
}

Example 17 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class TableTest method testMultiGetWithTx.

@Test
public void testMultiGetWithTx() throws Exception {
    String testMultiGet = "testMultiGet";
    DatasetAdmin admin = getTableAdmin(CONTEXT1, testMultiGet);
    admin.create();
    try (Table table = getTable(CONTEXT1, testMultiGet)) {
        Transaction tx = txClient.startShort();
        ((TransactionAware) table).startTx(tx);
        for (int i = 0; i < 100; i++) {
            table.put(new Put(Bytes.toBytes("r" + i)).add(C1, V1).add(C2, V2));
        }
        txClient.canCommitOrThrow(tx, ((TransactionAware) table).getTxChanges());
        Assert.assertTrue(((TransactionAware) table).commitTx());
        txClient.commitOrThrow(tx);
        Transaction tx2 = txClient.startShort();
        ((TransactionAware) table).startTx(tx2);
        List<Get> gets = Lists.newArrayListWithCapacity(100);
        for (int i = 0; i < 100; i++) {
            gets.add(new Get(Bytes.toBytes("r" + i)));
        }
        List<Row> results = table.get(gets);
        txClient.commitOrThrow(tx2);
        for (int i = 0; i < 100; i++) {
            Row row = results.get(i);
            Assert.assertArrayEquals(Bytes.toBytes("r" + i), row.getRow());
            byte[] val = row.get(C1);
            Assert.assertNotNull(val);
            Assert.assertArrayEquals(V1, val);
            byte[] val2 = row.get(C2);
            Assert.assertNotNull(val2);
            Assert.assertArrayEquals(V2, val2);
        }
        Transaction tx3 = txClient.startShort();
        ((TransactionAware) table).startTx(tx3);
        gets = Lists.newArrayListWithCapacity(100);
        for (int i = 0; i < 100; i++) {
            gets.add(new Get("r" + i).add(C1));
        }
        results = table.get(gets);
        txClient.commitOrThrow(tx3);
        for (int i = 0; i < 100; i++) {
            Row row = results.get(i);
            Assert.assertArrayEquals(Bytes.toBytes("r" + i), row.getRow());
            byte[] val = row.get(C1);
            Assert.assertNotNull(val);
            Assert.assertArrayEquals(V1, val);
            // should have only returned column 1
            byte[] val2 = row.get(C2);
            Assert.assertNull(val2);
        }
        // retrieve different columns per row
        Transaction tx4 = txClient.startShort();
        ((TransactionAware) table).startTx(tx4);
        gets = Lists.newArrayListWithCapacity(100);
        for (int i = 0; i < 100; i++) {
            Get get = new Get("r" + i);
            // evens get C1, odds get C2
            get.add(i % 2 == 0 ? C1 : C2);
            gets.add(get);
        }
        results = table.get(gets);
        txClient.commitOrThrow(tx4);
        for (int i = 0; i < 100; i++) {
            Row row = results.get(i);
            Assert.assertArrayEquals(Bytes.toBytes("r" + i), row.getRow());
            byte[] val1 = row.get(C1);
            byte[] val2 = row.get(C2);
            if (i % 2 == 0) {
                Assert.assertNotNull(val1);
                Assert.assertArrayEquals(V1, val1);
                Assert.assertNull(val2);
            } else {
                Assert.assertNull(val1);
                Assert.assertNotNull(val2);
                Assert.assertArrayEquals(V2, val2);
            }
        }
    } finally {
        admin.drop();
    }
}

Example 18 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class TableTest method testBasicDeleteWithTx.

@Test
public void testBasicDeleteWithTx() throws Exception {
    // we will test 3 different delete column ops and one delete row op
    // * delete column with delete
    // * delete column with put null value
    // * delete column with put byte[0] value
    // * delete row with delete
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try (Table myTable1 = getTable(CONTEXT1, MY_TABLE);
        Table myTable2 = getTable(CONTEXT1, MY_TABLE)) {
        // write smth and commit
        Transaction tx1 = txClient.startShort();
        ((TransactionAware) myTable1).startTx(tx1);
        myTable1.put(R1, a(C1, C2), a(V1, V2));
        myTable1.put(R2, a(C1, C2), a(V2, V3));
        myTable1.put(R3, a(C1, C2), a(V3, V4));
        myTable1.put(R4, a(C1, C2), a(V4, V5));
        txClient.canCommitOrThrow(tx1, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx1);
        // Now, we will test delete ops
        // start new tx2
        Transaction tx2 = txClient.startShort();
        ((TransactionAware) myTable2).startTx(tx2);
        // TableAssert.verify tx2 sees changes of tx1
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable2.get(R1, a(C1, C2)));
        // TableAssert.verify tx2 sees changes of tx1
        TableAssert.assertRow(a(C1, V2, C2, V3), myTable2.get(R2));
        // delete c1, r2
        myTable2.delete(R1, a(C1));
        myTable2.delete(R2);
        // same as delete a column
        myTable2.put(R3, C1, null);
        // TableAssert.verify can see deletes in own changes before commit
        TableAssert.assertRow(a(C2, V2), myTable2.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(null, myTable2.get(R1, C1));
        Assert.assertArrayEquals(V2, myTable2.get(R1, C2));
        TableAssert.assertRow(a(), myTable2.get(R2));
        TableAssert.assertRow(a(C2, V4), myTable2.get(R3));
        // overwrite c2 and write new value to c1
        myTable2.put(R1, a(C1, C2), a(V3, V4));
        myTable2.put(R2, a(C1, C2), a(V4, V5));
        myTable2.put(R3, a(C1, C2), a(V1, V2));
        myTable2.put(R4, a(C1, C2), a(V2, V3));
        // TableAssert.verify can see changes in own changes before commit
        TableAssert.assertRow(a(C1, V3, C2, V4), myTable2.get(R1, a(C1, C2, C3)));
        Assert.assertArrayEquals(V3, myTable2.get(R1, C1));
        Assert.assertArrayEquals(V4, myTable2.get(R1, C2));
        Assert.assertArrayEquals(null, myTable2.get(R1, C3));
        TableAssert.assertRow(a(C1, V4, C2, V5), myTable2.get(R2));
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable2.get(R3));
        TableAssert.assertRow(a(C1, V2, C2, V3), myTable2.get(R4));
        // delete c2 and r2
        myTable2.delete(R1, a(C2));
        myTable2.delete(R2);
        myTable2.put(R1, C2, null);
        // TableAssert.verify that delete is there (i.e. not reverted to whatever was persisted before)
        TableAssert.assertRow(a(C1, V3), myTable2.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(V3, myTable2.get(R1, C1));
        Assert.assertArrayEquals(null, myTable2.get(R1, C2));
        TableAssert.assertRow(a(), myTable2.get(R2));
        Assert.assertArrayEquals(V1, myTable2.get(R3, C1));
        Assert.assertArrayEquals(V2, myTable2.get(R4, C1));
        // start tx3 and TableAssert.verify that changes of tx2 are not visible yet
        Transaction tx3 = txClient.startShort();
        // NOTE: table instance can be re-used between tx
        ((TransactionAware) myTable1).startTx(tx3);
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(V1, myTable1.get(R1, C1));
        Assert.assertArrayEquals(V2, myTable1.get(R1, C2));
        Assert.assertArrayEquals(null, myTable1.get(R1, C3));
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1));
        TableAssert.assertRow(a(C1, V2, C2, V3), myTable1.get(R2));
        TableAssert.assertRow(a(C1, V3, C2, V4), myTable1.get(R3));
        TableAssert.assertRow(a(C1, V4, C2, V5), myTable1.get(R4));
        txClient.canCommitOrThrow(tx3, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx3);
        // starting tx4 before committing tx2 so that we can check conflicts are detected wrt deletes
        Transaction tx4 = txClient.startShort();
        // starting tx5 before committing tx2 so that we can check conflicts are detected wrt deletes
        Transaction tx5 = txClient.startShort();
        // commit tx2 in stages to see how races are handled wrt delete ops
        txClient.canCommitOrThrow(tx2, ((TransactionAware) myTable2).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable2).commitTx());
        // start tx6 and TableAssert.verify that changes of tx2 are not visible yet (even though they are flushed)
        Transaction tx6 = txClient.startShort();
        // NOTE: table instance can be re-used between tx
        ((TransactionAware) myTable1).startTx(tx6);
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(V1, myTable1.get(R1, C1));
        Assert.assertArrayEquals(V2, myTable1.get(R1, C2));
        Assert.assertArrayEquals(null, myTable1.get(R1, C3));
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1));
        TableAssert.assertRow(a(C1, V2, C2, V3), myTable1.get(R2));
        TableAssert.assertRow(a(C1, V3, C2, V4), myTable1.get(R3));
        TableAssert.assertRow(a(C1, V4, C2, V5), myTable1.get(R4));
        txClient.canCommitOrThrow(tx6, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx6);
        // make tx2 visible
        txClient.commitOrThrow(tx2);
        // start tx7 and TableAssert.verify that changes of tx2 are now visible
        Transaction tx7 = txClient.startShort();
        // NOTE: table instance can be re-used between tx
        ((TransactionAware) myTable1).startTx(tx7);
        TableAssert.assertRow(a(C1, V3), myTable1.get(R1, a(C1, C2)));
        TableAssert.assertRow(a(C1, V3), myTable1.get(R1));
        Assert.assertArrayEquals(V3, myTable1.get(R1, C1));
        Assert.assertArrayEquals(null, myTable1.get(R1, C2));
        TableAssert.assertRow(a(C1, V3), myTable1.get(R1, a(C1, C2)));
        TableAssert.assertRow(a(), myTable1.get(R2));
        Assert.assertArrayEquals(V1, myTable1.get(R3, C1));
        Assert.assertArrayEquals(V2, myTable1.get(R4, C1));
        txClient.canCommitOrThrow(tx6, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx7);
        // but not visible to tx4 that we started earlier than tx2 became visible
        // NOTE: table instance can be re-used between tx
        ((TransactionAware) myTable1).startTx(tx4);
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(V1, myTable1.get(R1, C1));
        Assert.assertArrayEquals(V2, myTable1.get(R1, C2));
        Assert.assertArrayEquals(null, myTable1.get(R1, C3));
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1));
        TableAssert.assertRow(a(C1, V2, C2, V3), myTable1.get(R2));
        TableAssert.assertRow(a(C1, V3, C2, V4), myTable1.get(R3));
        TableAssert.assertRow(a(C1, V4, C2, V5), myTable1.get(R4));
        // writing to deleted column, to check conflicts are detected (delete-write conflict)
        myTable1.put(R1, a(C2), a(V5));
        try {
            txClient.canCommitOrThrow(tx4, ((TransactionAware) myTable1).getTxChanges());
            Assert.fail("Conflict not detected!");
        } catch (TransactionConflictException e) {
        // expected
        }
        ((TransactionAware) myTable1).rollbackTx();
        txClient.abort(tx4);
        // deleting changed column, to check conflicts are detected (write-delete conflict)
        // NOTE: table instance can be re-used between tx
        ((TransactionAware) myTable1).startTx(tx5);
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(V1, myTable1.get(R1, C1));
        Assert.assertArrayEquals(V2, myTable1.get(R1, C2));
        Assert.assertArrayEquals(null, myTable1.get(R1, C3));
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable1.get(R1));
        TableAssert.assertRow(a(C1, V2, C2, V3), myTable1.get(R2));
        TableAssert.assertRow(a(C1, V3, C2, V4), myTable1.get(R3));
        TableAssert.assertRow(a(C1, V4, C2, V5), myTable1.get(R4));
        // NOTE: we are TableAssert.verifying conflict in one operation only. We may want to test each...
        myTable1.delete(R1, a(C1));
        try {
            txClient.canCommitOrThrow(tx5, ((TransactionAware) myTable1).getTxChanges());
            Assert.fail("Conflict not detected!");
        } catch (TransactionConflictException e) {
        // expected
        }
        ((TransactionAware) myTable1).rollbackTx();
        txClient.abort(tx5);
    } finally {
        admin.drop();
    }
}

Example 19 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class TableTest method testEmptyValuePut.

// TODO figure out what to do with this. As long as ObjectMappedTable writes empty values, we cannot
// throw exceptions, and this test is pointless.
@Ignore
@Test
public void testEmptyValuePut() throws Exception {
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    Transaction tx = txClient.startShort();
    try (Table myTable = getTable(CONTEXT1, MY_TABLE)) {
        try {
            myTable.put(R1, C1, MT);
            Assert.fail("Put with empty value should fail.");
        } catch (IllegalArgumentException e) {
        // expected
        }
        try {
            myTable.put(R1, a(C1, C2), a(V1, MT));
            Assert.fail("Put with empty value should fail.");
        } catch (IllegalArgumentException e) {
        // expected
        }
        try {
            myTable.put(new Put(R1).add(C1, V1).add(C2, MT));
            Assert.fail("Put with empty value should fail.");
        } catch (IllegalArgumentException e) {
        // expected
        }
        try {
            myTable.compareAndSwap(R1, C1, V1, MT);
            Assert.fail("CompareAndSwap with empty value should fail.");
        } catch (IllegalArgumentException e) {
        // expected
        }
    } finally {
        txClient.abort(tx);
        admin.drop();
    }
}

Example 20 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class TableTest method testMetrics.

private void testMetrics(boolean readless) throws Exception {
    final String tableName = "survive";
    DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(readless).build();
    DatasetAdmin admin = getTableAdmin(CONTEXT1, tableName, props);
    admin.create();
    try (Table table = getTable(CONTEXT1, tableName, props)) {
        final Map<String, Long> metrics = Maps.newHashMap();
        ((MeteredDataset) table).setMetricsCollector(new MetricsCollector() {

            @Override
            public void increment(String metricName, long value) {
                Long old = metrics.get(metricName);
                metrics.put(metricName, old == null ? value : old + value);
            }

            @Override
            public void gauge(String metricName, long value) {
                metrics.put(metricName, value);
            }
        });
        // Note that we don't need to finish tx for metrics to be reported
        Transaction tx0 = txClient.startShort();
        ((TransactionAware) table).startTx(tx0);
        int writes = 0;
        int reads = 0;
        table.put(new Put(R1, C1, V1));
        verifyDatasetMetrics(metrics, ++writes, reads);
        table.compareAndSwap(R1, C1, V1, V2);
        verifyDatasetMetrics(metrics, ++writes, ++reads);
        // note: will not write anything as expected value will not match
        table.compareAndSwap(R1, C1, V1, V2);
        verifyDatasetMetrics(metrics, writes, ++reads);
        table.increment(new Increment(R2, C2, 1L));
        if (readless) {
            verifyDatasetMetrics(metrics, ++writes, reads);
        } else {
            verifyDatasetMetrics(metrics, ++writes, ++reads);
        }
        table.incrementAndGet(new Increment(R2, C2, 1L));
        verifyDatasetMetrics(metrics, ++writes, ++reads);
        table.get(new Get(R1, C1, V1));
        verifyDatasetMetrics(metrics, writes, ++reads);
        Scanner scanner = table.scan(new Scan(null, null));
        while (scanner.next() != null) {
            verifyDatasetMetrics(metrics, writes, ++reads);
        }
        table.delete(new Delete(R1, C1, V1));
        verifyDatasetMetrics(metrics, ++writes, reads);
    } finally {
        // drop table
        admin.drop();
    }
}