Examples with DatasetProperties co.cask.cdap.api.dataset.DatasetProperties used on opensource projects

Example 6 with DatasetProperties

use of co.cask.cdap.api.dataset.DatasetProperties in project cdap by caskdata.

the class TableTest method testBasicIncrementWithTx.

private void testBasicIncrementWithTx(boolean doIncrAndGet, boolean readless) throws Exception {
    DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(readless).build();
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE, props);
    admin.create();
    Table myTable1, myTable2, myTable3, myTable4;
    try {
        Transaction tx1 = txClient.startShort();
        myTable1 = getTable(CONTEXT1, MY_TABLE, props);
        ((TransactionAware) myTable1).startTx(tx1);
        myTable1.put(R1, a(C1), a(L4));
        doIncrement(doIncrAndGet, myTable1, R1, a(C1), la(-3L), lb(1L));
        doIncrement(doIncrAndGet, myTable1, R1, a(C2), la(2L), lb(2L));
        // TableAssert.verify increment result visible inside tx before commit
        TableAssert.assertRow(a(C1, L1, C2, L2), myTable1.get(R1, a(C1, C2)));
        Assert.assertArrayEquals(L1, myTable1.get(R1, C1));
        Assert.assertArrayEquals(L2, myTable1.get(R1, C2));
        Assert.assertArrayEquals(null, myTable1.get(R1, C3));
        TableAssert.assertRow(a(C1, L1), myTable1.get(R1, a(C1)));
        TableAssert.assertRow(a(C1, L1, C2, L2), myTable1.get(R1));
        // incrementing non-long value should fail
        myTable1.put(R1, a(C5), a(V5));
        try {
            doIncrement(doIncrAndGet, myTable1, R1, a(C5), la(5L), lb(-1L));
            Assert.fail("increment should have failed with NumberFormatException");
        } catch (NumberFormatException e) {
        // Expected
        }
        // previous increment should not do any change
        TableAssert.assertRow(a(C5, V5), myTable1.get(R1, a(C5)));
        Assert.assertArrayEquals(V5, myTable1.get(R1, C5));
        // start new tx (doesn't see changes of the tx1)
        Transaction tx2 = txClient.startShort();
        // committing tx1 in stages to check races are handled well
        // * first, flush operations of table
        txClient.canCommitOrThrow(tx1, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        // check that tx2 doesn't see changes (even though they were flushed) of tx1
        // assuming current value is null
        myTable2 = getTable(CONTEXT1, MY_TABLE, props);
        ((TransactionAware) myTable2).startTx(tx2);
        TableAssert.assertRow(a(), myTable2.get(R1, a(C1, C2, C5)));
        Assert.assertArrayEquals(null, myTable2.get(R1, C1));
        Assert.assertArrayEquals(null, myTable2.get(R1, C2));
        Assert.assertArrayEquals(null, myTable2.get(R1, C5));
        TableAssert.assertRow(a(), myTable2.get(R1));
        doIncrement(doIncrAndGet, myTable2, R1, a(C1), la(55L), lb(55L));
        // start tx3 and TableAssert.verify same thing again
        Transaction tx3 = txClient.startShort();
        myTable3 = getTable(CONTEXT1, MY_TABLE, props);
        ((TransactionAware) myTable3).startTx(tx3);
        TableAssert.assertRow(a(), myTable3.get(R1, a(C1, C2, C5)));
        Assert.assertArrayEquals(null, myTable3.get(R1, C1));
        Assert.assertArrayEquals(null, myTable3.get(R1, C2));
        Assert.assertArrayEquals(null, myTable3.get(R1, C5));
        TableAssert.assertRow(a(), myTable3.get(R1));
        doIncrement(doIncrAndGet, myTable3, R1, a(C1), la(4L), lb(4L));
        // * second, make tx visible
        txClient.commitOrThrow(tx1);
        // TableAssert.verify that tx2 cannot commit because of the conflicts...
        try {
            txClient.canCommitOrThrow(tx2, ((TransactionAware) myTable2).getTxChanges());
            Assert.fail("Conflict not detected!");
        } catch (TransactionConflictException e) {
        // expected
        }
        ((TransactionAware) myTable2).rollbackTx();
        txClient.abort(tx2);
        // start tx4 and TableAssert.verify that changes of tx1 are now visible
        Transaction tx4 = txClient.startShort();
        myTable4 = getTable(CONTEXT1, MY_TABLE, props);
        ((TransactionAware) myTable4).startTx(tx4);
        TableAssert.assertRow(a(C1, L1, C2, L2, C5, V5), myTable4.get(R1, a(C1, C2, C3, C4, C5)));
        TableAssert.assertRow(a(C2, L2), myTable4.get(R1, a(C2)));
        Assert.assertArrayEquals(L1, myTable4.get(R1, C1));
        Assert.assertArrayEquals(L2, myTable4.get(R1, C2));
        Assert.assertArrayEquals(null, myTable4.get(R1, C3));
        Assert.assertArrayEquals(V5, myTable4.get(R1, C5));
        TableAssert.assertRow(a(C1, L1, C5, V5), myTable4.get(R1, a(C1, C5)));
        TableAssert.assertRow(a(C1, L1, C2, L2, C5, V5), myTable4.get(R1));
        // tx3 still cannot see tx1 changes, only its own
        TableAssert.assertRow(a(C1, L4), myTable3.get(R1, a(C1, C2, C5)));
        Assert.assertArrayEquals(L4, myTable3.get(R1, C1));
        Assert.assertArrayEquals(null, myTable3.get(R1, C2));
        Assert.assertArrayEquals(null, myTable3.get(R1, C5));
        TableAssert.assertRow(a(C1, L4), myTable3.get(R1));
        // and it cannot commit because its changes cause conflicts
        try {
            txClient.canCommitOrThrow(tx3, ((TransactionAware) myTable3).getTxChanges());
            Assert.fail("Conflict not detected!");
        } catch (TransactionConflictException e) {
        // expected
        }
        ((TransactionAware) myTable3).rollbackTx();
        txClient.abort(tx3);
        // TableAssert.verify we can do some ops with tx4 based on data written with tx1
        doIncrement(doIncrAndGet, myTable4, R1, a(C1, C2, C3), la(2L, 1L, 5L), lb(3L, 3L, 5L));
        myTable4.delete(R1, a(C2));
        doIncrement(doIncrAndGet, myTable4, R1, a(C4), la(3L), lb(3L));
        myTable4.delete(R1, a(C1));
        // committing tx4
        txClient.canCommitOrThrow(tx4, ((TransactionAware) myTable3).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable4).commitTx());
        txClient.commitOrThrow(tx4);
        // TableAssert.verifying the result contents in next transaction
        Transaction tx5 = txClient.startShort();
        // NOTE: table instance can be re-used in series of transactions
        ((TransactionAware) myTable4).startTx(tx5);
        TableAssert.assertRow(a(C3, L5, C4, L3, C5, V5), myTable4.get(R1, a(C1, C2, C3, C4, C5)));
        Assert.assertArrayEquals(null, myTable4.get(R1, C1));
        Assert.assertArrayEquals(null, myTable4.get(R1, C2));
        Assert.assertArrayEquals(L5, myTable4.get(R1, C3));
        Assert.assertArrayEquals(L3, myTable4.get(R1, C4));
        Assert.assertArrayEquals(V5, myTable4.get(R1, C5));
        TableAssert.assertRow(a(C3, L5, C4, L3, C5, V5), myTable4.get(R1));
        txClient.canCommitOrThrow(tx5, ((TransactionAware) myTable3).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable3).commitTx());
        txClient.commitOrThrow(tx5);
    } finally {
        admin.drop();
    }
}

Example 7 with DatasetProperties

use of co.cask.cdap.api.dataset.DatasetProperties in project cdap by caskdata.

the class BufferingTableTest method testChangingParamsAndReturnValues.

@Test
public void testChangingParamsAndReturnValues() throws Exception {
    // The test verifies that one can re-use byte arrays passed as parameters to write methods of a table without
    // affecting the stored data.
    // Also, one can re-use (modify) returned data from the table without affecting the stored data.
    DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(isReadlessIncrementSupported()).build();
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE, props);
    admin.create();
    try {
        // writing some data: we'll need it to test delete later
        Transaction tx = txClient.startShort();
        BufferingTable table = getTable(CONTEXT1, MY_TABLE, props);
        table.startTx(tx);
        table.put(new byte[] { 0 }, new byte[] { 9 }, new byte[] { 8 });
        table.commitTx();
        txClient.commitOrThrow(tx);
        // start new for in-mem buffer behavior testing
        tx = txClient.startShort();
        table.startTx(tx);
        // write some data but not commit
        byte[] rowParam = new byte[] { 1 };
        byte[] colParam = new byte[] { 2 };
        byte[] valParam = Bytes.toBytes(3L);
        table.put(rowParam, colParam, valParam);
        verify123(table);
        // change passed earlier byte arrays in place, this should not affect stored previously values
        rowParam[0]++;
        colParam[0]++;
        valParam[0]++;
        verify123(table);
        // try get row and change returned values in place, which should not affect the data stored
        Row getRow = table.get(new byte[] { 1 });
        Map<byte[], byte[]> getRowResult = getRow.getColumns();
        Assert.assertEquals(1, getRowResult.size());
        byte[] colFromGetRow = getRowResult.keySet().iterator().next();
        byte[] valFromGetRow = getRowResult.get(colFromGetRow);
        getRowResult.remove(new byte[] { 2 });
        Assert.assertArrayEquals(new byte[] { 2 }, colFromGetRow);
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromGetRow);
        colFromGetRow[0]++;
        valFromGetRow[0]++;
        verify123(table);
        // try get set of columns in a row and change returned values in place, which should not affect the data stored
        Row getColumnSetRow = table.get(new byte[] { 1 });
        Map<byte[], byte[]> getColumnSetResult = getColumnSetRow.getColumns();
        Assert.assertEquals(1, getColumnSetResult.size());
        byte[] colFromGetColumnSet = getColumnSetResult.keySet().iterator().next();
        byte[] valFromGetColumnSet = getColumnSetResult.values().iterator().next();
        getColumnSetResult.remove(new byte[] { 2 });
        Assert.assertArrayEquals(new byte[] { 2 }, colFromGetColumnSet);
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromGetColumnSet);
        colFromGetColumnSet[0]++;
        valFromGetColumnSet[0]++;
        verify123(table);
        // try get column and change returned value in place, which should not affect the data stored
        byte[] valFromGetColumn = table.get(new byte[] { 1 }, new byte[] { 2 });
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromGetColumn);
        valFromGetColumn[0]++;
        verify123(table);
        // try scan and change returned value in place, which should not affect the data stored
        Scanner scan = table.scan(new byte[] { 1 }, null);
        Row next = scan.next();
        Assert.assertNotNull(next);
        byte[] rowFromScan = next.getRow();
        Assert.assertArrayEquals(new byte[] { 1 }, rowFromScan);
        Map<byte[], byte[]> cols = next.getColumns();
        Assert.assertEquals(1, cols.size());
        byte[] colFromScan = cols.keySet().iterator().next();
        Assert.assertArrayEquals(new byte[] { 2 }, colFromScan);
        byte[] valFromScan = next.get(new byte[] { 2 });
        Assert.assertNotNull(valFromScan);
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromScan);
        Assert.assertNull(scan.next());
        cols.remove(new byte[] { 2 });
        rowFromScan[0]++;
        colFromScan[0]++;
        valFromScan[0]++;
        verify123(table);
        // try delete and change params in place: this should not affect stored data
        rowParam = new byte[] { 1 };
        colParam = new byte[] { 2 };
        table.delete(rowParam, colParam);
        Assert.assertNull(table.get(new byte[] { 1 }, new byte[] { 2 }));
        Assert.assertArrayEquals(new byte[] { 8 }, table.get(new byte[] { 0 }, new byte[] { 9 }));
        rowParam[0] = 0;
        colParam[0] = 9;
        Assert.assertNull(table.get(new byte[] { 1 }, new byte[] { 2 }));
        Assert.assertArrayEquals(new byte[] { 8 }, table.get(new byte[] { 0 }, new byte[] { 9 }));
        // try increment column and change params in place: this should not affect stored data
        byte[] rowIncParam = new byte[] { 1 };
        byte[] colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, 3);
        verify123(table);
        rowIncParam[0]++;
        colIncParam[0]++;
        verify123(table);
        // try increment set of columns and change params in place, try also to change values in returned map: this all
        // should not affect stored data.
        rowIncParam = new byte[] { 1 };
        colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, -1);
        table.increment(rowIncParam, new byte[][] { colIncParam }, new long[] { 1 });
        verify123(table);
        rowIncParam[0]++;
        colIncParam[0]++;
        verify123(table);
        // try increment and change returned values: should not affect the stored data
        rowIncParam = new byte[] { 1 };
        colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, -1);
        Row countersRow = table.incrementAndGet(rowIncParam, new byte[][] { colIncParam }, new long[] { 1 });
        Map<byte[], byte[]> counters = countersRow.getColumns();
        Assert.assertEquals(1, counters.size());
        byte[] colFromInc = counters.keySet().iterator().next();
        Assert.assertArrayEquals(new byte[] { 2 }, colFromInc);
        Assert.assertEquals(3, Bytes.toLong(counters.get(colFromInc)));
        counters.remove(new byte[] { 2 });
        colFromInc[0]++;
        verify123(table);
        // try increment write and change params in place: this should not affect stored data
        rowIncParam = new byte[] { 1 };
        colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, -1);
        table.increment(rowIncParam, new byte[][] { colIncParam }, new long[] { 1 });
        verify123(table);
        rowIncParam[0]++;
        colIncParam[0]++;
        verify123(table);
        // try compareAndSwap and change params in place: this should not affect stored data
        byte[] rowSwapParam = new byte[] { 1 };
        byte[] colSwapParam = new byte[] { 2 };
        byte[] valSwapParam = Bytes.toBytes(3L);
        table.compareAndSwap(rowSwapParam, colSwapParam, Bytes.toBytes(3L), Bytes.toBytes(4L));
        table.compareAndSwap(rowSwapParam, colSwapParam, Bytes.toBytes(4L), valSwapParam);
        verify123(table);
        rowSwapParam[0]++;
        colSwapParam[0]++;
        valSwapParam[0]++;
        verify123(table);
    // We don't care to persist changes and commit tx here: we tested what we wanted
    } finally {
        admin.drop();
    }
}

Example 8 with DatasetProperties

use of co.cask.cdap.api.dataset.DatasetProperties in project cdap by caskdata.

the class CubeDatasetDefinition method configure.

@Override
public DatasetSpecification configure(String instanceName, DatasetProperties properties) {
    DatasetProperties factTableProperties = computeFactTableProperties(properties);
    List<DatasetSpecification> datasetSpecs = Lists.newArrayList();
    // Configuring table that hold mappings of tag names and values and such
    datasetSpecs.add(metricsTableDef.configure("entity", properties));
    // NOTE: we create a table per resolution; we later will use that to e.g. configure ttl separately for each
    for (int resolution : getResolutions(properties.getProperties())) {
        datasetSpecs.add(tableDef.configure(String.valueOf(resolution), factTableProperties));
    }
    return DatasetSpecification.builder(instanceName, getName()).properties(properties.getProperties()).datasets(datasetSpecs).build();
}

Example 9 with DatasetProperties

use of co.cask.cdap.api.dataset.DatasetProperties in project cdap by caskdata.

the class HBaseTableTest method testScannerCache.

private void testScannerCache(int rowsExpected, String tableName, @Nullable String property, @Nullable String argument, @Nullable String scanArgument) throws Exception {
    // Now scan and sleep for a while after each result
    Transaction tx = txClient.startShort();
    DatasetProperties props = property == null ? DatasetProperties.EMPTY : DatasetProperties.of(ImmutableMap.of(HConstants.HBASE_CLIENT_SCANNER_CACHING, property));
    Map<String, String> arguments = argument == null ? Collections.<String, String>emptyMap() : ImmutableMap.of(HConstants.HBASE_CLIENT_SCANNER_CACHING, argument);
    Scan scan = new Scan(null, null);
    if (scanArgument != null) {
        scan.setProperty(HConstants.HBASE_CLIENT_SCANNER_CACHING, scanArgument);
    }
    Table table = getTable(CONTEXT1, tableName, props, arguments);
    ((TransactionAware) table).startTx(tx);
    Scanner scanner = table.scan(scan);
    int scanCount = 0;
    try {
        while (scanner.next() != null) {
            scanCount++;
            TimeUnit.MILLISECONDS.sleep(10);
        }
        scanner.close();
    } finally {
        LOG.info("Scanned {} rows.", scanCount);
        txClient.abort(tx);
    }
    Assert.assertEquals(rowsExpected, scanCount);
}

Example 10 with DatasetProperties

use of co.cask.cdap.api.dataset.DatasetProperties in project cdap by caskdata.

the class HBaseTableTest method testCachedEncodedTransaction.

@Test
public void testCachedEncodedTransaction() throws Exception {
    String tableName = "testEncodedTxTable";
    DatasetProperties props = DatasetProperties.EMPTY;
    getTableAdmin(CONTEXT1, tableName, props).create();
    DatasetSpecification tableSpec = DatasetSpecification.builder(tableName, HBaseTable.class.getName()).build();
    // use a transaction codec that counts the number of times encode() is called
    final AtomicInteger encodeCount = new AtomicInteger();
    final TransactionCodec codec = new TransactionCodec() {

        @Override
        public byte[] encode(Transaction tx) throws IOException {
            encodeCount.incrementAndGet();
            return super.encode(tx);
        }
    };
    // use a table util that creates an HTable that validates the encoded tx on each get
    final AtomicReference<Transaction> txRef = new AtomicReference<>();
    HBaseTableUtil util = new DelegatingHBaseTableUtil(hBaseTableUtil) {

        @Override
        public HTable createHTable(Configuration conf, TableId tableId) throws IOException {
            HTable htable = super.createHTable(conf, tableId);
            return new MinimalDelegatingHTable(htable) {

                @Override
                public Result get(org.apache.hadoop.hbase.client.Get get) throws IOException {
                    Assert.assertEquals(txRef.get().getTransactionId(), codec.decode(get.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY)).getTransactionId());
                    return super.get(get);
                }

                @Override
                public Result[] get(List<org.apache.hadoop.hbase.client.Get> gets) throws IOException {
                    for (org.apache.hadoop.hbase.client.Get get : gets) {
                        Assert.assertEquals(txRef.get().getTransactionId(), codec.decode(get.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY)).getTransactionId());
                    }
                    return super.get(gets);
                }

                @Override
                public ResultScanner getScanner(org.apache.hadoop.hbase.client.Scan scan) throws IOException {
                    Assert.assertEquals(txRef.get().getTransactionId(), codec.decode(scan.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY)).getTransactionId());
                    return super.getScanner(scan);
                }
            };
        }
    };
    HBaseTable table = new HBaseTable(CONTEXT1, tableSpec, Collections.<String, String>emptyMap(), cConf, TEST_HBASE.getConfiguration(), util, codec);
    DetachedTxSystemClient txSystemClient = new DetachedTxSystemClient();
    // test all operations: only the first one encodes
    Transaction tx = txSystemClient.startShort();
    txRef.set(tx);
    table.startTx(tx);
    table.put(b("row1"), b("col1"), b("val1"));
    Assert.assertEquals(0, encodeCount.get());
    table.get(b("row"));
    Assert.assertEquals(1, encodeCount.get());
    table.get(ImmutableList.of(new Get("a"), new Get("b")));
    Assert.assertEquals(1, encodeCount.get());
    Scanner scanner = table.scan(new Scan(null, null));
    Assert.assertEquals(1, encodeCount.get());
    scanner.close();
    table.increment(b("z"), b("z"), 0L);
    Assert.assertEquals(1, encodeCount.get());
    table.commitTx();
    table.postTxCommit();
    // test that for the next tx, we encode again
    tx = txSystemClient.startShort();
    txRef.set(tx);
    table.startTx(tx);
    table.get(b("row"));
    Assert.assertEquals(2, encodeCount.get());
    table.commitTx();
    // test that we encode again, even of postTxCommit was not called
    tx = txSystemClient.startShort();
    txRef.set(tx);
    table.startTx(tx);
    table.get(b("row"));
    Assert.assertEquals(3, encodeCount.get());
    table.commitTx();
    table.rollbackTx();
    // test that rollback does not encode the tx
    Assert.assertEquals(3, encodeCount.get());
    // test that we encode again if the previous tx rolled back
    tx = txSystemClient.startShort();
    txRef.set(tx);
    table.startTx(tx);
    table.get(b("row"));
    Assert.assertEquals(4, encodeCount.get());
    table.commitTx();
    table.close();
    Assert.assertEquals(4, encodeCount.get());
}