Examples with DatasetAdmin io.cdap.cdap.api.dataset.DatasetAdmin used on opensource projects

Example 36 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by cdapio.

the class HBaseTableTest method testEnforceTxLifetime.

@Test
public void testEnforceTxLifetime() throws Exception {
    String tableName = "enforce-tx-lifetime";
    DatasetProperties datasetProperties = TableProperties.builder().setReadlessIncrementSupport(true).setConflictDetection(ConflictDetection.COLUMN).build();
    DatasetAdmin admin = getTableAdmin(CONTEXT1, tableName, datasetProperties);
    admin.create();
    DetachedTxSystemClient txSystemClient = new DetachedTxSystemClient();
    DatasetSpecification spec = DatasetSpecification.builder(tableName, HBaseTable.class.getName()).properties(datasetProperties.getProperties()).build();
    try {
        final HBaseTable table = new HBaseTable(CONTEXT1, spec, Collections.<String, String>emptyMap(), cConf, TEST_HBASE.getConfiguration(), hBaseTableUtil);
        Transaction tx = txSystemClient.startShort();
        table.startTx(tx);
        table.put(b("row1"), b("col1"), b("val1"));
        table.put(b("inc1"), b("col1"), Bytes.toBytes(10L));
        table.commitTx();
        table.postTxCommit();
        table.close();
        CConfiguration testCConf = CConfiguration.copy(cConf);
        // No mutations on tables using testCConf will succeed.
        testCConf.setInt(TxConstants.Manager.CFG_TX_MAX_LIFETIME, 0);
        try (final HBaseTable failTable = new HBaseTable(CONTEXT1, spec, Collections.<String, String>emptyMap(), testCConf, TEST_HBASE.getConfiguration(), hBaseTableUtil)) {
            // A put should fail
            assertTxFail(txSystemClient, failTable, new Runnable() {

                @Override
                public void run() {
                    failTable.put(b("row2"), b("col1"), b("val1"));
                }
            });
            // A delete should also fail
            assertTxFail(txSystemClient, failTable, new Runnable() {

                @Override
                public void run() {
                    failTable.delete(b("row1"));
                }
            });
            assertTxFail(txSystemClient, failTable, new Runnable() {

                @Override
                public void run() {
                    failTable.delete(b("row1"), b("col1"));
                }
            });
            // So should an increment
            assertTxFail(txSystemClient, failTable, new Runnable() {

                @Override
                public void run() {
                    failTable.increment(b("inc1"), b("col1"), 10);
                }
            });
            // incrementAndGet gets converted to a put internally
            assertTxFail(txSystemClient, failTable, new Runnable() {

                @Override
                public void run() {
                    failTable.incrementAndGet(b("inc1"), b("col1"), 10);
                }
            });
        }
        // Even safe increments should fail (this happens when readless increment is done from a mapreduce job)
        try (final HBaseTable failTable = new HBaseTable(CONTEXT1, spec, ImmutableMap.of(HBaseTable.SAFE_INCREMENTS, "true"), testCConf, TEST_HBASE.getConfiguration(), hBaseTableUtil)) {
            // So should an increment
            assertTxFail(txSystemClient, failTable, new Runnable() {

                @Override
                public void run() {
                    failTable.increment(b("inc1"), b("col1"), 10);
                }
            });
            // incrementAndGet gets converted to a put internally
            assertTxFail(txSystemClient, failTable, new Runnable() {

                @Override
                public void run() {
                    failTable.incrementAndGet(b("inc1"), b("col1"), 10);
                }
            });
        }
    } finally {
        admin.drop();
        admin.close();
    }
}

Example 37 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by cdapio.

the class BufferingTableTest method testScanWithBuffering.

/**
 * Tests that writes being buffered in memory by the client are still visible during scans.
 */
@Test
public void testScanWithBuffering() throws Exception {
    String testScanWithBuffering = "testScanWithBuffering";
    DatasetAdmin admin = getTableAdmin(CONTEXT1, testScanWithBuffering);
    admin.create();
    try (Table table1 = getTable(CONTEXT1, testScanWithBuffering)) {
        // 
        Transaction tx1 = txClient.startShort();
        ((TransactionAware) table1).startTx(tx1);
        table1.put(Bytes.toBytes("1_01"), a(C1), a(V1));
        table1.put(Bytes.toBytes("1_02"), a(C1), a(V1));
        table1.put(Bytes.toBytes("1_03"), a(C1), a(V1));
        // written values should not yet be persisted
        TableAssert.assertScan(new byte[0][], new byte[0][][], ((BufferingTable) table1).scanPersisted(new Scan(Bytes.toBytes("1_"), Bytes.toBytes("2_"))));
        // buffered values should be visible in a scan
        TableAssert.assertScan(a(Bytes.toBytes("1_01"), Bytes.toBytes("1_02"), Bytes.toBytes("1_03")), aa(a(C1, V1), a(C1, V1), a(C1, V1)), table1.scan(Bytes.toBytes("1_"), Bytes.toBytes("2_")));
        txClient.canCommitOrThrow(tx1, ((TransactionAware) table1).getTxChanges());
        Assert.assertTrue(((TransactionAware) table1).commitTx());
        txClient.commitOrThrow(tx1);
        Transaction tx2 = txClient.startShort();
        ((TransactionAware) table1).startTx(tx2);
        // written values should be visible after commit
        TableAssert.assertScan(a(Bytes.toBytes("1_01"), Bytes.toBytes("1_02"), Bytes.toBytes("1_03")), aa(a(C1, V1), a(C1, V1), a(C1, V1)), table1.scan(Bytes.toBytes("1_"), Bytes.toBytes("2_")));
        txClient.commitOrThrow(tx2);
        Transaction tx3 = txClient.startShort();
        ((TransactionAware) table1).startTx(tx3);
        // test merging of buffered writes on existing rows
        table1.put(Bytes.toBytes("1_01"), a(C2), a(V2));
        table1.put(Bytes.toBytes("1_02"), a(C1), a(V2));
        table1.put(Bytes.toBytes("1_02a"), a(C1), a(V1));
        table1.put(Bytes.toBytes("1_02b"), a(C1), a(V1));
        table1.put(Bytes.toBytes("1_04"), a(C2), a(V2));
        // persisted values should be the same
        TableAssert.assertScan(a(Bytes.toBytes("1_01"), Bytes.toBytes("1_02"), Bytes.toBytes("1_03")), aa(a(C1, V1), a(C1, V1), a(C1, V1)), ((BufferingTable) table1).scanPersisted(new Scan(Bytes.toBytes("1_"), Bytes.toBytes("2_"))));
        // all values should be visible in buffered scan
        TableAssert.assertScan(a(Bytes.toBytes("1_01"), Bytes.toBytes("1_02"), Bytes.toBytes("1_02a"), Bytes.toBytes("1_02b"), Bytes.toBytes("1_03"), Bytes.toBytes("1_04")), aa(// 1_01
        a(C1, V1, C2, V2), // 1_02
        a(C1, V2), // 1_02a
        a(C1, V1), // 1_02b
        a(C1, V1), // 1_03
        a(C1, V1), // 1_04
        a(C2, V2)), table1.scan(Bytes.toBytes("1_"), Bytes.toBytes("2_")));
        txClient.canCommitOrThrow(tx3, ((TransactionAware) table1).getTxChanges());
        Assert.assertTrue(((TransactionAware) table1).commitTx());
        txClient.commitOrThrow(tx3);
        Transaction tx4 = txClient.startShort();
        ((TransactionAware) table1).startTx(tx4);
        // all values should be visible after commit
        TableAssert.assertScan(a(Bytes.toBytes("1_01"), Bytes.toBytes("1_02"), Bytes.toBytes("1_02a"), Bytes.toBytes("1_02b"), Bytes.toBytes("1_03"), Bytes.toBytes("1_04")), aa(// 1_01
        a(C1, V1, C2, V2), // 1_02
        a(C1, V2), // 1_02a
        a(C1, V1), // 1_02b
        a(C1, V1), // 1_03
        a(C1, V1), // 1_04
        a(C2, V2)), table1.scan(Bytes.toBytes("1_"), Bytes.toBytes("2_")));
        txClient.commitOrThrow(tx4);
    } finally {
        admin.drop();
    }
}

Example 38 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by cdapio.

the class BufferingTableTest method testRollingBackAfterExceptionDuringPersist.

@Test
public void testRollingBackAfterExceptionDuringPersist() throws Exception {
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try (BufferingTable myTable1 = new BufferingTableWithPersistingFailure(getTable(CONTEXT1, MY_TABLE))) {
        Transaction tx1 = txClient.startShort();
        myTable1.startTx(tx1);
        // write some data but not commit
        myTable1.put(R1, a(C1), a(V1));
        myTable1.put(R2, a(C2), a(V2));
        // verify can see changes inside tx
        TableAssert.assertRow(a(C1, V1), myTable1.get(R1, a(C1)));
        TableAssert.assertRow(a(C2, V2), myTable1.get(R2, a(C2)));
        // persisting changes
        try {
            // should simulate exception
            myTable1.commitTx();
            Assert.assertFalse(true);
        } catch (Throwable th) {
        // Expected simulated exception
        }
        // let's pretend that after persisting changes we still got conflict when finalizing tx, so
        // rolling back changes
        Assert.assertTrue(myTable1.rollbackTx());
        // making tx visible
        txClient.abort(tx1);
        // start new tx
        Transaction tx2 = txClient.startShort();
        try (Table myTable2 = getTable(CONTEXT1, MY_TABLE)) {
            ((TransactionAware) myTable2).startTx(tx2);
            // verify don't see rolled back changes
            TableAssert.assertRow(a(), myTable2.get(R1, a(C1)));
            TableAssert.assertRow(a(), myTable2.get(R2, a(C2)));
        }
    } finally {
        admin.drop();
    }
}

Example 39 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by cdapio.

the class TableTest method testEmptyGet.

@Test
public void testEmptyGet() throws Exception {
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try (Table myTable = getTable(CONTEXT1, MY_TABLE)) {
        Transaction tx = txClient.startShort();
        ((TransactionAware) myTable).startTx(tx);
        myTable.put(R1, C1, V1);
        myTable.put(R1, C2, V2);
        // to be used for validation later
        TreeMap<byte[], byte[]> expectedColumns = new TreeMap<>(Bytes.BYTES_COMPARATOR);
        expectedColumns.put(C1, V1);
        expectedColumns.put(C2, V2);
        Result expectedResult = new Result(R1, expectedColumns);
        Result emptyResult = new Result(R1, ImmutableMap.<byte[], byte[]>of());
        ((TransactionAware) myTable).commitTx();
        txClient.commitOrThrow(tx);
        // start another transaction, so that the buffering table doesn't cache the values; the underlying Table
        // implementations are tested this way.
        tx = txClient.startShort();
        ((TransactionAware) myTable).startTx(tx);
        Row row = myTable.get(R1, new byte[][] { C1, C2 });
        assertEquals(expectedResult, row);
        // passing in empty columns returns empty result
        row = myTable.get(R1, new byte[][] {});
        assertEquals(emptyResult, row);
        // test all the Get constructors and their behavior
        // constructors specifying only rowkey retrieve all columns
        Get get = new Get(R1);
        Assert.assertNull(get.getColumns());
        assertEquals(expectedResult, myTable.get(get));
        get = new Get(Bytes.toString(R1));
        Assert.assertNull(get.getColumns());
        assertEquals(expectedResult, myTable.get(get));
        get.add(C1);
        get.add(Bytes.toString(C2));
        assertEquals(expectedResult, myTable.get(get));
        // constructor specifying columns, but with an empty array/collection retrieve 0 columns
        get = new Get(R1, new byte[][] {});
        Assert.assertNotNull(get.getColumns());
        assertEquals(emptyResult, myTable.get(get));
        get = new Get(R1, ImmutableList.<byte[]>of());
        Assert.assertNotNull(get.getColumns());
        assertEquals(emptyResult, myTable.get(get));
        get = new Get(Bytes.toString(R1), new String[] {});
        Assert.assertNotNull(get.getColumns());
        assertEquals(emptyResult, myTable.get(get));
        get = new Get(Bytes.toString(R1), ImmutableList.<String>of());
        Assert.assertNotNull(get.getColumns());
        assertEquals(emptyResult, myTable.get(get));
        row = myTable.get(R1, new byte[][] {});
        assertEquals(emptyResult, row);
        txClient.abort(tx);
    } finally {
        admin.drop();
    }
}

Example 40 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by cdapio.

the class TableTest method testBasicIncrementWriteWithTxSmall.

private void testBasicIncrementWriteWithTxSmall(boolean readless) throws Exception {
    DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(readless).build();
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE, props);
    admin.create();
    try (Table myTable = getTable(CONTEXT1, MY_TABLE, props)) {
        // start 1st tx
        Transaction tx = txClient.startShort();
        ((TransactionAware) myTable).startTx(tx);
        myTable.increment(R1, a(C1), la(-3L));
        // we'll use this one to test that we can delete increment and increment again
        myTable.increment(R2, a(C2), la(5L));
        commitAndAssertSuccess(tx, (TransactionAware) myTable);
        // start 2nd tx
        tx = txClient.startShort();
        ((TransactionAware) myTable).startTx(tx);
        Assert.assertArrayEquals(Bytes.toBytes(-3L), myTable.get(R1, C1));
        myTable.increment(R1, a(C1), la(-3L));
        Assert.assertArrayEquals(Bytes.toBytes(-6L), myTable.get(R1, C1));
        Assert.assertArrayEquals(Bytes.toBytes(5L), myTable.get(R2, C2));
        myTable.delete(R2, C2);
        Assert.assertArrayEquals(null, myTable.get(R2, C2));
        commitAndAssertSuccess(tx, (TransactionAware) myTable);
        // start 3rd tx
        tx = txClient.startShort();
        ((TransactionAware) myTable).startTx(tx);
        Assert.assertArrayEquals(Bytes.toBytes(-6L), myTable.get(R1, C1));
        Assert.assertArrayEquals(null, myTable.get(R2, C2));
        myTable.increment(R2, a(C2), la(7L));
        Assert.assertArrayEquals(Bytes.toBytes(7L), myTable.get(R2, C2));
        commitAndAssertSuccess(tx, (TransactionAware) myTable);
        // start 4rd tx
        tx = txClient.startShort();
        ((TransactionAware) myTable).startTx(tx);
        Assert.assertArrayEquals(Bytes.toBytes(7L), myTable.get(R2, C2));
        commitAndAssertSuccess(tx, (TransactionAware) myTable);
    } finally {
        admin.drop();
    }
}