Examples with DatasetAdmin io.cdap.cdap.api.dataset.DatasetAdmin used on opensource projects

Example 56 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class TableTest method testBasicColumnRangeWithTx.

@Test
public void testBasicColumnRangeWithTx() throws Exception {
    // todo: test more tx logic (or add to get/put unit-test)
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try (Table myTable1 = getTable(CONTEXT1, MY_TABLE);
        Table myTable2 = getTable(CONTEXT1, MY_TABLE)) {
        // write test data and commit
        Transaction tx1 = txClient.startShort();
        ((TransactionAware) myTable1).startTx(tx1);
        myTable1.put(R1, a(C1, C2, C3, C4, C5), a(V1, V2, V3, V4, V5));
        myTable1.put(R2, a(C1), a(V2));
        txClient.canCommitOrThrow(tx1, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx1);
        // Now, we will test column range get
        Transaction tx2 = txClient.startShort();
        ((TransactionAware) myTable2).startTx(tx2);
        // bounded range
        TableAssert.assertRow(a(C2, V2, C3, V3, C4, V4), myTable2.get(R1, C2, C5, Integer.MAX_VALUE));
        // open start range
        TableAssert.assertRow(a(C1, V1, C2, V2, C3, V3), myTable2.get(R1, null, C4, Integer.MAX_VALUE));
        // open end range
        TableAssert.assertRow(a(C3, V3, C4, V4, C5, V5), myTable2.get(R1, C3, null, Integer.MAX_VALUE));
        // open ends range
        TableAssert.assertRow(a(C1, V1, C2, V2, C3, V3, C4, V4, C5, V5), myTable2.get(R1, null, null, Integer.MAX_VALUE));
        // same with limit
        // bounded range with limit
        TableAssert.assertRow(a(C2, V2), myTable2.get(R1, C2, C5, 1));
        // open start range with limit
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable2.get(R1, null, C4, 2));
        // open end range with limit
        TableAssert.assertRow(a(C3, V3, C4, V4), myTable2.get(R1, C3, null, 2));
        // open ends range with limit
        TableAssert.assertRow(a(C1, V1, C2, V2, C3, V3, C4, V4), myTable2.get(R1, null, null, 4));
        // adding/changing/removing some columns
        myTable2.put(R1, a(C1, C2, C3), a(V4, V3, V2));
        myTable2.delete(R1, a(C4));
        txClient.canCommitOrThrow(tx2, ((TransactionAware) myTable2).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable2).commitTx());
        txClient.commitOrThrow(tx2);
        // Checking that changes are reflected in new scans in new tx
        Transaction tx3 = txClient.startShort();
        // NOTE: table can be re-used betweet tx
        ((TransactionAware) myTable1).startTx(tx3);
        TableAssert.assertRow(a(C2, V3, C3, V2), myTable1.get(R1, C2, C5, Integer.MAX_VALUE));
        txClient.canCommitOrThrow(tx3, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx3);
    } finally {
        admin.drop();
    }
}

Example 57 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class TableTest method testEmptyGet.

@Test
public void testEmptyGet() throws Exception {
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try (Table myTable = getTable(CONTEXT1, MY_TABLE)) {
        Transaction tx = txClient.startShort();
        ((TransactionAware) myTable).startTx(tx);
        myTable.put(R1, C1, V1);
        myTable.put(R1, C2, V2);
        // to be used for validation later
        TreeMap<byte[], byte[]> expectedColumns = new TreeMap<>(Bytes.BYTES_COMPARATOR);
        expectedColumns.put(C1, V1);
        expectedColumns.put(C2, V2);
        Result expectedResult = new Result(R1, expectedColumns);
        Result emptyResult = new Result(R1, ImmutableMap.<byte[], byte[]>of());
        ((TransactionAware) myTable).commitTx();
        txClient.commitOrThrow(tx);
        // start another transaction, so that the buffering table doesn't cache the values; the underlying Table
        // implementations are tested this way.
        tx = txClient.startShort();
        ((TransactionAware) myTable).startTx(tx);
        Row row = myTable.get(R1, new byte[][] { C1, C2 });
        assertEquals(expectedResult, row);
        // passing in empty columns returns empty result
        row = myTable.get(R1, new byte[][] {});
        assertEquals(emptyResult, row);
        // test all the Get constructors and their behavior
        // constructors specifying only rowkey retrieve all columns
        Get get = new Get(R1);
        Assert.assertNull(get.getColumns());
        assertEquals(expectedResult, myTable.get(get));
        get = new Get(Bytes.toString(R1));
        Assert.assertNull(get.getColumns());
        assertEquals(expectedResult, myTable.get(get));
        get.add(C1);
        get.add(Bytes.toString(C2));
        assertEquals(expectedResult, myTable.get(get));
        // constructor specifying columns, but with an empty array/collection retrieve 0 columns
        get = new Get(R1, new byte[][] {});
        Assert.assertNotNull(get.getColumns());
        assertEquals(emptyResult, myTable.get(get));
        get = new Get(R1, ImmutableList.<byte[]>of());
        Assert.assertNotNull(get.getColumns());
        assertEquals(emptyResult, myTable.get(get));
        get = new Get(Bytes.toString(R1), new String[] {});
        Assert.assertNotNull(get.getColumns());
        assertEquals(emptyResult, myTable.get(get));
        get = new Get(Bytes.toString(R1), ImmutableList.<String>of());
        Assert.assertNotNull(get.getColumns());
        assertEquals(emptyResult, myTable.get(get));
        row = myTable.get(R1, new byte[][] {});
        assertEquals(emptyResult, row);
        txClient.abort(tx);
    } finally {
        admin.drop();
    }
}

Example 58 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class TableTest method testBasicScanWithTx.

@Test
public void testBasicScanWithTx() throws Exception {
    // todo: make work with tx well (merge with buffer, conflicts) and add tests for that
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try (Table myTable1 = getTable(CONTEXT1, MY_TABLE);
        Table myTable2 = getTable(CONTEXT1, MY_TABLE)) {
        // write r1...r5 and commit
        Transaction tx1 = txClient.startShort();
        ((TransactionAware) myTable1).startTx(tx1);
        myTable1.put(R1, a(C1), a(V1));
        myTable1.put(R2, a(C2), a(V2));
        myTable1.put(R3, a(C3, C4), a(V3, V4));
        myTable1.put(R4, a(C4), a(V4));
        myTable1.put(R5, a(C5), a(V5));
        txClient.canCommitOrThrow(tx1, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx1);
        // Now, we will test scans
        // currently not testing races/conflicts/etc as this logic is not there for scans yet; so using one same tx
        Transaction tx2 = txClient.startShort();
        ((TransactionAware) myTable2).startTx(tx2);
        // bounded scan
        TableAssert.assertScan(a(R2, R3, R4), aa(a(C2, V2), a(C3, V3, C4, V4), a(C4, V4)), myTable2, new Scan(R2, R5));
        // open start scan
        TableAssert.assertScan(a(R1, R2, R3), aa(a(C1, V1), a(C2, V2), a(C3, V3, C4, V4)), myTable2, new Scan(null, R4));
        // open end scan
        TableAssert.assertScan(a(R3, R4, R5), aa(a(C3, V3, C4, V4), a(C4, V4), a(C5, V5)), myTable2, new Scan(R3, null));
        // open ends scan
        TableAssert.assertScan(a(R1, R2, R3, R4, R5), aa(a(C1, V1), a(C2, V2), a(C3, V3, C4, V4), a(C4, V4), a(C5, V5)), myTable2, new Scan(null, null));
        // adding/changing/removing some columns
        myTable2.put(R2, a(C1, C2, C3), a(V4, V3, V2));
        myTable2.delete(R3, a(C4));
        txClient.canCommitOrThrow(tx2, ((TransactionAware) myTable2).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable2).commitTx());
        txClient.commitOrThrow(tx2);
        // Checking that changes are reflected in new scans in new tx
        Transaction tx3 = txClient.startShort();
        // NOTE: table can be re-used betweet tx
        ((TransactionAware) myTable1).startTx(tx3);
        TableAssert.assertScan(a(R2, R3, R4), aa(a(C1, V4, C2, V3, C3, V2), a(C3, V3), a(C4, V4)), myTable1, new Scan(R2, R5));
        txClient.canCommitOrThrow(tx3, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx3);
    } finally {
        admin.drop();
    }
}

Example 59 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class TableTest method testScanWithFuzzyRowFilter.

@Test
public void testScanWithFuzzyRowFilter() throws Exception {
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try (Table table = getTable(CONTEXT1, MY_TABLE)) {
        Transaction tx1 = txClient.startShort();
        ((TransactionAware) table).startTx(tx1);
        // write data
        byte[] abc = { 'a', 'b', 'c' };
        for (byte b1 : abc) {
            for (byte b2 : abc) {
                for (byte b3 : abc) {
                    for (byte b4 : abc) {
                        table.put(new Put(new byte[] { b1, b2, b3, b4 }).add(C1, V1));
                    }
                }
            }
        }
        // we should have 81 (3^4) rows now
        Assert.assertEquals(81, countRows(table));
        // check that filter works against data written in same tx
        verifyScanWithFuzzyRowFilter(table);
        // commit tx, start new and TableAssert.verify scan again against "persisted" data
        txClient.canCommitOrThrow(tx1, ((TransactionAware) table).getTxChanges());
        Assert.assertTrue(((TransactionAware) table).commitTx());
        txClient.commitOrThrow(tx1);
        ((TransactionAware) table).postTxCommit();
        Transaction tx2 = txClient.startShort();
        ((TransactionAware) table).startTx(tx2);
        verifyScanWithFuzzyRowFilter(table);
    } finally {
        admin.drop();
    }
}

Example 60 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class TableTest method testBasicCompareAndSwapWithTx.

@Test
public void testBasicCompareAndSwapWithTx() throws Exception {
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try (Table myTable1 = getTable(CONTEXT1, MY_TABLE);
        Table myTable2 = getTable(CONTEXT1, MY_TABLE);
        Table myTable3 = getTable(CONTEXT1, MY_TABLE);
        Table myTable4 = getTable(CONTEXT1, MY_TABLE)) {
        Transaction tx1 = txClient.startShort();
        ((TransactionAware) myTable1).startTx(tx1);
        // write r1->c1,v1 but not commit
        myTable1.put(R1, a(C1), a(V1));
        // write r1->c2,v2 but not commit
        Assert.assertTrue(myTable1.compareAndSwap(R1, C2, null, V5));
        // TableAssert.verify compare and swap result visible inside tx before commit
        TableAssert.assertRow(a(C1, V1, C2, V5), myTable1.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(V1, myTable1.get(R1, C1));
        Assert.assertArrayEquals(V5, myTable1.get(R1, C2));
        Assert.assertArrayEquals(null, myTable1.get(R1, C3));
        TableAssert.assertRow(a(C1, V1, C2, V5), myTable1.get(R1));
        // these should fail
        Assert.assertFalse(myTable1.compareAndSwap(R1, C1, null, V1));
        Assert.assertFalse(myTable1.compareAndSwap(R1, C1, V2, V1));
        Assert.assertFalse(myTable1.compareAndSwap(R1, C2, null, V2));
        Assert.assertFalse(myTable1.compareAndSwap(R1, C2, V2, V1));
        // but this should succeed
        Assert.assertTrue(myTable1.compareAndSwap(R1, C2, V5, V2));
        // start new tx (doesn't see changes of the tx1)
        Transaction tx2 = txClient.startShort();
        // committing tx1 in stages to check races are handled well
        // * first, flush operations of table
        txClient.canCommitOrThrow(tx1, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        // check that tx2 doesn't see changes (even though they were flushed) of tx1 by trying to compareAndSwap
        // assuming current value is null
        ((TransactionAware) myTable2).startTx(tx2);
        Assert.assertTrue(myTable2.compareAndSwap(R1, C1, null, V3));
        // start tx3 and TableAssert.verify same thing again
        Transaction tx3 = txClient.startShort();
        ((TransactionAware) myTable3).startTx(tx3);
        Assert.assertTrue(myTable3.compareAndSwap(R1, C1, null, V2));
        // * second, make tx visible
        txClient.commitOrThrow(tx1);
        // TableAssert.verify that tx2 cannot commit because of the conflicts...
        try {
            txClient.canCommitOrThrow(tx2, ((TransactionAware) myTable2).getTxChanges());
            Assert.fail("Conflict not detected!");
        } catch (TransactionConflictException e) {
        // expected
        }
        ((TransactionAware) myTable2).rollbackTx();
        txClient.abort(tx2);
        // start tx4 and TableAssert.verify that changes of tx1 are now visible
        Transaction tx4 = txClient.startShort();
        ((TransactionAware) myTable4).startTx(tx4);
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable4.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(V1, myTable4.get(R1, C1));
        Assert.assertArrayEquals(V2, myTable4.get(R1, C2));
        Assert.assertArrayEquals(null, myTable4.get(R1, C3));
        TableAssert.assertRow(a(C2, V2), myTable4.get(R1, a(C2)));
        TableAssert.assertRow(a(C1, V1, C2, V2), myTable4.get(R1));
        // tx3 still cannot see tx1 changes
        Assert.assertTrue(myTable3.compareAndSwap(R1, C2, null, V5));
        // and it cannot commit because its changes cause conflicts
        try {
            txClient.canCommitOrThrow(tx3, ((TransactionAware) myTable3).getTxChanges());
            Assert.fail("Conflict not detected!");
        } catch (TransactionConflictException e) {
        // expected
        }
        ((TransactionAware) myTable3).rollbackTx();
        txClient.abort(tx3);
        // TableAssert.verify we can do some ops with tx4 based on data written with tx1
        Assert.assertFalse(myTable4.compareAndSwap(R1, C1, null, V4));
        Assert.assertFalse(myTable4.compareAndSwap(R1, C2, null, V5));
        Assert.assertTrue(myTable4.compareAndSwap(R1, C1, V1, V3));
        Assert.assertTrue(myTable4.compareAndSwap(R1, C2, V2, V4));
        myTable4.delete(R1, a(C1));
        // committing tx4
        txClient.canCommitOrThrow(tx4, ((TransactionAware) myTable3).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable4).commitTx());
        txClient.commitOrThrow(tx4);
        // TableAssert.verifying the result contents in next transaction
        Transaction tx5 = txClient.startShort();
        // NOTE: table instance can be re-used in series of transactions
        ((TransactionAware) myTable4).startTx(tx5);
        TableAssert.assertRow(a(C2, V4), myTable4.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(null, myTable4.get(R1, C1));
        Assert.assertArrayEquals(V4, myTable4.get(R1, C2));
        TableAssert.assertRow(a(C2, V4), myTable4.get(R1));
        txClient.canCommitOrThrow(tx5, ((TransactionAware) myTable3).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable3).commitTx());
        txClient.commitOrThrow(tx5);
    } finally {
        admin.drop();
    }
}