Examples with DatasetAdmin co.cask.cdap.api.dataset.DatasetAdmin used on opensource projects

Example 6 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class BufferingTableTest method testChangingParamsAndReturnValues.

@Test
public void testChangingParamsAndReturnValues() throws Exception {
    // The test verifies that one can re-use byte arrays passed as parameters to write methods of a table without
    // affecting the stored data.
    // Also, one can re-use (modify) returned data from the table without affecting the stored data.
    DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(isReadlessIncrementSupported()).build();
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE, props);
    admin.create();
    try {
        // writing some data: we'll need it to test delete later
        Transaction tx = txClient.startShort();
        BufferingTable table = getTable(CONTEXT1, MY_TABLE, props);
        table.startTx(tx);
        table.put(new byte[] { 0 }, new byte[] { 9 }, new byte[] { 8 });
        table.commitTx();
        txClient.commit(tx);
        // start new for in-mem buffer behavior testing
        tx = txClient.startShort();
        table.startTx(tx);
        // write some data but not commit
        byte[] rowParam = new byte[] { 1 };
        byte[] colParam = new byte[] { 2 };
        byte[] valParam = Bytes.toBytes(3L);
        table.put(rowParam, colParam, valParam);
        verify123(table);
        // change passed earlier byte arrays in place, this should not affect stored previously values
        rowParam[0]++;
        colParam[0]++;
        valParam[0]++;
        verify123(table);
        // try get row and change returned values in place, which should not affect the data stored
        Row getRow = table.get(new byte[] { 1 });
        Map<byte[], byte[]> getRowResult = getRow.getColumns();
        Assert.assertEquals(1, getRowResult.size());
        byte[] colFromGetRow = getRowResult.keySet().iterator().next();
        byte[] valFromGetRow = getRowResult.get(colFromGetRow);
        getRowResult.remove(new byte[] { 2 });
        Assert.assertArrayEquals(new byte[] { 2 }, colFromGetRow);
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromGetRow);
        colFromGetRow[0]++;
        valFromGetRow[0]++;
        verify123(table);
        // try get set of columns in a row and change returned values in place, which should not affect the data stored
        Row getColumnSetRow = table.get(new byte[] { 1 });
        Map<byte[], byte[]> getColumnSetResult = getColumnSetRow.getColumns();
        Assert.assertEquals(1, getColumnSetResult.size());
        byte[] colFromGetColumnSet = getColumnSetResult.keySet().iterator().next();
        byte[] valFromGetColumnSet = getColumnSetResult.values().iterator().next();
        getColumnSetResult.remove(new byte[] { 2 });
        Assert.assertArrayEquals(new byte[] { 2 }, colFromGetColumnSet);
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromGetColumnSet);
        colFromGetColumnSet[0]++;
        valFromGetColumnSet[0]++;
        verify123(table);
        // try get column and change returned value in place, which should not affect the data stored
        byte[] valFromGetColumn = table.get(new byte[] { 1 }, new byte[] { 2 });
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromGetColumn);
        valFromGetColumn[0]++;
        verify123(table);
        // try scan and change returned value in place, which should not affect the data stored
        Scanner scan = table.scan(new byte[] { 1 }, null);
        Row next = scan.next();
        Assert.assertNotNull(next);
        byte[] rowFromScan = next.getRow();
        Assert.assertArrayEquals(new byte[] { 1 }, rowFromScan);
        Map<byte[], byte[]> cols = next.getColumns();
        Assert.assertEquals(1, cols.size());
        byte[] colFromScan = cols.keySet().iterator().next();
        Assert.assertArrayEquals(new byte[] { 2 }, colFromScan);
        byte[] valFromScan = next.get(new byte[] { 2 });
        Assert.assertNotNull(valFromScan);
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromScan);
        Assert.assertNull(scan.next());
        cols.remove(new byte[] { 2 });
        rowFromScan[0]++;
        colFromScan[0]++;
        valFromScan[0]++;
        verify123(table);
        // try delete and change params in place: this should not affect stored data
        rowParam = new byte[] { 1 };
        colParam = new byte[] { 2 };
        table.delete(rowParam, colParam);
        Assert.assertNull(table.get(new byte[] { 1 }, new byte[] { 2 }));
        Assert.assertArrayEquals(new byte[] { 8 }, table.get(new byte[] { 0 }, new byte[] { 9 }));
        rowParam[0] = 0;
        colParam[0] = 9;
        Assert.assertNull(table.get(new byte[] { 1 }, new byte[] { 2 }));
        Assert.assertArrayEquals(new byte[] { 8 }, table.get(new byte[] { 0 }, new byte[] { 9 }));
        // try increment column and change params in place: this should not affect stored data
        byte[] rowIncParam = new byte[] { 1 };
        byte[] colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, 3);
        verify123(table);
        rowIncParam[0]++;
        colIncParam[0]++;
        verify123(table);
        // try increment set of columns and change params in place, try also to change values in returned map: this all
        // should not affect stored data.
        rowIncParam = new byte[] { 1 };
        colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, -1);
        table.increment(rowIncParam, new byte[][] { colIncParam }, new long[] { 1 });
        verify123(table);
        rowIncParam[0]++;
        colIncParam[0]++;
        verify123(table);
        // try increment and change returned values: should not affect the stored data
        rowIncParam = new byte[] { 1 };
        colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, -1);
        Row countersRow = table.incrementAndGet(rowIncParam, new byte[][] { colIncParam }, new long[] { 1 });
        Map<byte[], byte[]> counters = countersRow.getColumns();
        Assert.assertEquals(1, counters.size());
        byte[] colFromInc = counters.keySet().iterator().next();
        Assert.assertArrayEquals(new byte[] { 2 }, colFromInc);
        Assert.assertEquals(3, Bytes.toLong(counters.get(colFromInc)));
        counters.remove(new byte[] { 2 });
        colFromInc[0]++;
        verify123(table);
        // try increment write and change params in place: this should not affect stored data
        rowIncParam = new byte[] { 1 };
        colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, -1);
        table.increment(rowIncParam, new byte[][] { colIncParam }, new long[] { 1 });
        verify123(table);
        rowIncParam[0]++;
        colIncParam[0]++;
        verify123(table);
        // try compareAndSwap and change params in place: this should not affect stored data
        byte[] rowSwapParam = new byte[] { 1 };
        byte[] colSwapParam = new byte[] { 2 };
        byte[] valSwapParam = Bytes.toBytes(3L);
        table.compareAndSwap(rowSwapParam, colSwapParam, Bytes.toBytes(3L), Bytes.toBytes(4L));
        table.compareAndSwap(rowSwapParam, colSwapParam, Bytes.toBytes(4L), valSwapParam);
        verify123(table);
        rowSwapParam[0]++;
        colSwapParam[0]++;
        valSwapParam[0]++;
        verify123(table);
    // We don't care to persist changes and commit tx here: we tested what we wanted
    } finally {
        admin.drop();
    }
}

Example 7 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class BufferingTableTest method testRollingBackAfterExceptionDuringPersist.

@Test
public void testRollingBackAfterExceptionDuringPersist() throws Exception {
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try {
        Transaction tx1 = txClient.startShort();
        BufferingTable myTable1 = new BufferingTableWithPersistingFailure(getTable(CONTEXT1, MY_TABLE));
        myTable1.startTx(tx1);
        // write some data but not commit
        myTable1.put(R1, a(C1), a(V1));
        myTable1.put(R2, a(C2), a(V2));
        // verify can see changes inside tx
        TableAssert.assertRow(a(C1, V1), myTable1.get(R1, a(C1)));
        TableAssert.assertRow(a(C2, V2), myTable1.get(R2, a(C2)));
        // persisting changes
        try {
            // should simulate exception
            myTable1.commitTx();
            Assert.assertFalse(true);
        } catch (Throwable th) {
        // Expected simulated exception
        }
        // let's pretend that after persisting changes we still got conflict when finalizing tx, so
        // rolling back changes
        Assert.assertTrue(myTable1.rollbackTx());
        // making tx visible
        txClient.abort(tx1);
        // start new tx
        Transaction tx2 = txClient.startShort();
        Table myTable2 = getTable(CONTEXT1, MY_TABLE);
        ((TransactionAware) myTable2).startTx(tx2);
        // verify don't see rolled back changes
        TableAssert.assertRow(a(), myTable2.get(R1, a(C1)));
        TableAssert.assertRow(a(), myTable2.get(R2, a(C2)));
    } finally {
        admin.drop();
    }
}

Example 8 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class BufferingTableTest method testScanWithBuffering.

/**
   * Tests that writes being buffered in memory by the client are still visible during scans.
   */
@Test
public void testScanWithBuffering() throws Exception {
    String testScanWithBuffering = "testScanWithBuffering";
    DatasetAdmin admin = getTableAdmin(CONTEXT1, testScanWithBuffering);
    admin.create();
    try {
        //
        Transaction tx1 = txClient.startShort();
        Table table1 = getTable(CONTEXT1, testScanWithBuffering);
        ((TransactionAware) table1).startTx(tx1);
        table1.put(Bytes.toBytes("1_01"), a(C1), a(V1));
        table1.put(Bytes.toBytes("1_02"), a(C1), a(V1));
        table1.put(Bytes.toBytes("1_03"), a(C1), a(V1));
        // written values should not yet be persisted
        TableAssert.assertScan(new byte[0][], new byte[0][][], ((BufferingTable) table1).scanPersisted(new Scan(Bytes.toBytes("1_"), Bytes.toBytes("2_"))));
        // buffered values should be visible in a scan
        TableAssert.assertScan(a(Bytes.toBytes("1_01"), Bytes.toBytes("1_02"), Bytes.toBytes("1_03")), aa(a(C1, V1), a(C1, V1), a(C1, V1)), table1.scan(Bytes.toBytes("1_"), Bytes.toBytes("2_")));
        Assert.assertTrue(txClient.canCommit(tx1, ((TransactionAware) table1).getTxChanges()));
        Assert.assertTrue(((TransactionAware) table1).commitTx());
        Assert.assertTrue(txClient.commit(tx1));
        Transaction tx2 = txClient.startShort();
        ((TransactionAware) table1).startTx(tx2);
        // written values should be visible after commit
        TableAssert.assertScan(a(Bytes.toBytes("1_01"), Bytes.toBytes("1_02"), Bytes.toBytes("1_03")), aa(a(C1, V1), a(C1, V1), a(C1, V1)), table1.scan(Bytes.toBytes("1_"), Bytes.toBytes("2_")));
        txClient.commit(tx2);
        Transaction tx3 = txClient.startShort();
        ((TransactionAware) table1).startTx(tx3);
        // test merging of buffered writes on existing rows
        table1.put(Bytes.toBytes("1_01"), a(C2), a(V2));
        table1.put(Bytes.toBytes("1_02"), a(C1), a(V2));
        table1.put(Bytes.toBytes("1_02a"), a(C1), a(V1));
        table1.put(Bytes.toBytes("1_02b"), a(C1), a(V1));
        table1.put(Bytes.toBytes("1_04"), a(C2), a(V2));
        // persisted values should be the same
        TableAssert.assertScan(a(Bytes.toBytes("1_01"), Bytes.toBytes("1_02"), Bytes.toBytes("1_03")), aa(a(C1, V1), a(C1, V1), a(C1, V1)), ((BufferingTable) table1).scanPersisted(new Scan(Bytes.toBytes("1_"), Bytes.toBytes("2_"))));
        // all values should be visible in buffered scan
        TableAssert.assertScan(a(Bytes.toBytes("1_01"), Bytes.toBytes("1_02"), Bytes.toBytes("1_02a"), Bytes.toBytes("1_02b"), Bytes.toBytes("1_03"), Bytes.toBytes("1_04")), aa(// 1_01
        a(C1, V1, C2, V2), // 1_02
        a(C1, V2), // 1_02a
        a(C1, V1), // 1_02b
        a(C1, V1), // 1_03
        a(C1, V1), // 1_04
        a(C2, V2)), table1.scan(Bytes.toBytes("1_"), Bytes.toBytes("2_")));
        Assert.assertTrue(txClient.canCommit(tx3, ((TransactionAware) table1).getTxChanges()));
        Assert.assertTrue(((TransactionAware) table1).commitTx());
        txClient.commit(tx3);
        Transaction tx4 = txClient.startShort();
        ((TransactionAware) table1).startTx(tx4);
        // all values should be visible after commit
        TableAssert.assertScan(a(Bytes.toBytes("1_01"), Bytes.toBytes("1_02"), Bytes.toBytes("1_02a"), Bytes.toBytes("1_02b"), Bytes.toBytes("1_03"), Bytes.toBytes("1_04")), aa(// 1_01
        a(C1, V1, C2, V2), // 1_02
        a(C1, V2), // 1_02a
        a(C1, V1), // 1_02b
        a(C1, V1), // 1_03
        a(C1, V1), // 1_04
        a(C2, V2)), table1.scan(Bytes.toBytes("1_"), Bytes.toBytes("2_")));
        txClient.commit(tx4);
    } finally {
        admin.drop();
    }
}

Example 9 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class TableTest method testConflictDetection.

private void testConflictDetection(ConflictDetection level) throws Exception {
    // we use tableX_Y format for variable names which means "tableX that is used in tx Y"
    String table1 = "table1";
    String table2 = "table2";
    DatasetProperties props = TableProperties.builder().setConflictDetection(level).build();
    DatasetAdmin admin1 = getTableAdmin(CONTEXT1, table1, props);
    DatasetAdmin admin2 = getTableAdmin(CONTEXT1, table2, props);
    admin1.create();
    admin2.create();
    try {
        // 1) Test conflicts when using different tables
        Transaction tx1 = txClient.startShort();
        Table table11 = getTable(CONTEXT1, table1, props);
        ((TransactionAware) table11).startTx(tx1);
        // write r1->c1,v1 but not commit
        table11.put(R1, a(C1), a(V1));
        // start new tx
        Transaction tx2 = txClient.startShort();
        Table table22 = getTable(CONTEXT1, table2, props);
        ((TransactionAware) table22).startTx(tx2);
        // change in tx2 same data but in different table
        table22.put(R1, a(C1), a(V2));
        // start new tx
        Transaction tx3 = txClient.startShort();
        Table table13 = getTable(CONTEXT1, table1, props);
        ((TransactionAware) table13).startTx(tx3);
        // change in tx3 same data in same table as tx1
        table13.put(R1, a(C1), a(V2));
        // committing tx1
        Assert.assertTrue(txClient.canCommit(tx1, ((TransactionAware) table11).getTxChanges()));
        Assert.assertTrue(((TransactionAware) table11).commitTx());
        Assert.assertTrue(txClient.commit(tx1));
        // no conflict should be when committing tx2
        Assert.assertTrue(txClient.canCommit(tx2, ((TransactionAware) table22).getTxChanges()));
        // but conflict should be when committing tx3
        if (level != ConflictDetection.NONE) {
            Assert.assertFalse(txClient.canCommit(tx3, ((TransactionAware) table13).getTxChanges()));
            ((TransactionAware) table13).rollbackTx();
            txClient.abort(tx3);
        } else {
            Assert.assertTrue(txClient.canCommit(tx3, ((TransactionAware) table13).getTxChanges()));
        }
        // 2) Test conflicts when using different rows
        Transaction tx4 = txClient.startShort();
        Table table14 = getTable(CONTEXT1, table1, props);
        ((TransactionAware) table14).startTx(tx4);
        // write r1->c1,v1 but not commit
        table14.put(R1, a(C1), a(V1));
        // start new tx
        Transaction tx5 = txClient.startShort();
        Table table15 = getTable(CONTEXT1, table1, props);
        ((TransactionAware) table15).startTx(tx5);
        // change in tx5 same data but in different row
        table15.put(R2, a(C1), a(V2));
        // start new tx
        Transaction tx6 = txClient.startShort();
        Table table16 = getTable(CONTEXT1, table1, props);
        ((TransactionAware) table16).startTx(tx6);
        // change in tx6 in same row as tx1
        table16.put(R1, a(C2), a(V2));
        // committing tx4
        Assert.assertTrue(txClient.canCommit(tx4, ((TransactionAware) table14).getTxChanges()));
        Assert.assertTrue(((TransactionAware) table14).commitTx());
        Assert.assertTrue(txClient.commit(tx4));
        // no conflict should be when committing tx5
        Assert.assertTrue(txClient.canCommit(tx5, ((TransactionAware) table15).getTxChanges()));
        // but conflict should be when committing tx6 iff we resolve on row level
        if (level == ConflictDetection.ROW) {
            Assert.assertFalse(txClient.canCommit(tx6, ((TransactionAware) table16).getTxChanges()));
            ((TransactionAware) table16).rollbackTx();
            txClient.abort(tx6);
        } else {
            Assert.assertTrue(txClient.canCommit(tx6, ((TransactionAware) table16).getTxChanges()));
        }
        // 3) Test conflicts when using different columns
        Transaction tx7 = txClient.startShort();
        Table table17 = getTable(CONTEXT1, table1, props);
        ((TransactionAware) table17).startTx(tx7);
        // write r1->c1,v1 but not commit
        table17.put(R1, a(C1), a(V1));
        // start new tx
        Transaction tx8 = txClient.startShort();
        Table table18 = getTable(CONTEXT1, table1, props);
        ((TransactionAware) table18).startTx(tx8);
        // change in tx8 same data but in different column
        table18.put(R1, a(C2), a(V2));
        // start new tx
        Transaction tx9 = txClient.startShort();
        Table table19 = getTable(CONTEXT1, table1, props);
        ((TransactionAware) table19).startTx(tx9);
        // change in tx9 same column in same column as tx1
        table19.put(R1, a(C1), a(V2));
        // committing tx7
        Assert.assertTrue(txClient.canCommit(tx7, ((TransactionAware) table17).getTxChanges()));
        Assert.assertTrue(((TransactionAware) table17).commitTx());
        Assert.assertTrue(txClient.commit(tx7));
        // no conflict should be when committing tx8 iff we resolve on column level
        if (level == ConflictDetection.COLUMN || level == ConflictDetection.NONE) {
            Assert.assertTrue(txClient.canCommit(tx8, ((TransactionAware) table18).getTxChanges()));
        } else {
            Assert.assertFalse(txClient.canCommit(tx8, ((TransactionAware) table18).getTxChanges()));
            ((TransactionAware) table18).rollbackTx();
            txClient.abort(tx8);
        }
        // but conflict should be when committing tx9
        if (level != ConflictDetection.NONE) {
            Assert.assertFalse(txClient.canCommit(tx9, ((TransactionAware) table19).getTxChanges()));
            ((TransactionAware) table19).rollbackTx();
            txClient.abort(tx9);
        } else {
            Assert.assertTrue(txClient.canCommit(tx9, ((TransactionAware) table19).getTxChanges()));
        }
    } finally {
        // NOTE: we are doing our best to cleanup junk between tests to isolate errors, but we are not going to be
        //       crazy about it
        admin1.drop();
        admin2.drop();
    }
}

Example 10 with DatasetAdmin

use of co.cask.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class TableTest method testEmptyGet.

@Test
public void testEmptyGet() throws Exception {
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try {
        Transaction tx = txClient.startShort();
        Table myTable = getTable(CONTEXT1, MY_TABLE);
        ((TransactionAware) myTable).startTx(tx);
        myTable.put(R1, C1, V1);
        myTable.put(R1, C2, V2);
        // to be used for validation later
        TreeMap<byte[], byte[]> expectedColumns = new TreeMap<>(Bytes.BYTES_COMPARATOR);
        expectedColumns.put(C1, V1);
        expectedColumns.put(C2, V2);
        Result expectedResult = new Result(R1, expectedColumns);
        Result emptyResult = new Result(R1, ImmutableMap.<byte[], byte[]>of());
        ((TransactionAware) myTable).commitTx();
        txClient.commit(tx);
        // start another transaction, so that the buffering table doesn't cache the values; the underlying Table
        // implementations are tested this way.
        tx = txClient.startShort();
        ((TransactionAware) myTable).startTx(tx);
        Row row = myTable.get(R1, new byte[][] { C1, C2 });
        assertEquals(expectedResult, row);
        // passing in empty columns returns empty result
        row = myTable.get(R1, new byte[][] {});
        assertEquals(emptyResult, row);
        // test all the Get constructors and their behavior
        // constructors specifying only rowkey retrieve all columns
        Get get = new Get(R1);
        Assert.assertNull(get.getColumns());
        assertEquals(expectedResult, myTable.get(get));
        get = new Get(Bytes.toString(R1));
        Assert.assertNull(get.getColumns());
        assertEquals(expectedResult, myTable.get(get));
        get.add(C1);
        get.add(Bytes.toString(C2));
        assertEquals(expectedResult, myTable.get(get));
        // constructor specifying columns, but with an empty array/collection retrieve 0 columns
        get = new Get(R1, new byte[][] {});
        Assert.assertNotNull(get.getColumns());
        assertEquals(emptyResult, myTable.get(get));
        get = new Get(R1, ImmutableList.<byte[]>of());
        Assert.assertNotNull(get.getColumns());
        assertEquals(emptyResult, myTable.get(get));
        get = new Get(Bytes.toString(R1), new String[] {});
        Assert.assertNotNull(get.getColumns());
        assertEquals(emptyResult, myTable.get(get));
        get = new Get(Bytes.toString(R1), ImmutableList.<String>of());
        Assert.assertNotNull(get.getColumns());
        assertEquals(emptyResult, myTable.get(get));
        row = myTable.get(R1, new byte[][] {});
        assertEquals(emptyResult, row);
        txClient.abort(tx);
    } finally {
        admin.drop();
    }
}