Search in sources :

Example 66 with TransactionExecutor

use of org.apache.tephra.TransactionExecutor in project cdap by caskdata.

the class MetadataStoreDatasetTest method testList.

@Test
public void testList() throws Exception {
    DatasetId storeTable = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("testList");
    dsFrameworkUtil.createInstance(Table.class.getName(), storeTable, DatasetProperties.EMPTY);
    Table table = dsFrameworkUtil.getInstance(storeTable);
    Assert.assertNotNull(table);
    final MetadataStoreDataset metadataStoreDataset = new MetadataStoreDataset(table);
    TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor((TransactionAware) table);
    final Map<MDSKey, Integer> expectedMap = new HashMap<>();
    final Map<MDSKey, Integer> expectedMapHalf = new HashMap<>();
    // Write some values
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            for (int i = 0; i < 5; ++i) {
                MDSKey mdsKey = new MDSKey.Builder().add(i).build();
                metadataStoreDataset.write(mdsKey, i);
                expectedMap.put(mdsKey, i);
                if ((i % 2) == 0) {
                    expectedMapHalf.put(mdsKey, i);
                }
            }
        }
    });
    // Fetch one record at a time
    for (int i = 0; i < 5; ++i) {
        final int fv = i;
        txnl.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                Map<MDSKey, Integer> val = metadataStoreDataset.listKV(new MDSKey.Builder().add(0).build(), new MDSKey.Builder().add(5).build(), Integer.class, 1, new Predicate<Integer>() {

                    @Override
                    public boolean apply(Integer input) {
                        return input == fv;
                    }
                });
                Assert.assertEquals(1, val.size());
                Assert.assertEquals(fv, (int) Iterables.get(val.values(), 0));
            }
        });
    }
    // Fetch two records at a time
    for (int i = 0; i < 4; ++i) {
        final int fv = i;
        txnl.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                Map<MDSKey, Integer> val = metadataStoreDataset.listKV(new MDSKey.Builder().add(0).build(), new MDSKey.Builder().add(5).build(), Integer.class, 2, new Predicate<Integer>() {

                    @Override
                    public boolean apply(Integer input) {
                        return input == fv || input == fv + 1;
                    }
                });
                Assert.assertEquals(2, val.size());
                Assert.assertEquals(fv, (int) Iterables.get(val.values(), 0));
                Assert.assertEquals(fv + 1, (int) Iterables.get(val.values(), 1));
            }
        });
    }
    // Fetch all keys using keySet
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Map<MDSKey, Integer> val = metadataStoreDataset.listKV(expectedMap.keySet(), Integer.class, 5);
            Assert.assertEquals(expectedMap, val);
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Map<MDSKey, Integer> valHalf = metadataStoreDataset.listKV(expectedMapHalf.keySet(), Integer.class, 5);
            Assert.assertEquals(expectedMapHalf, valHalf);
        }
    });
}
Also used : Table(co.cask.cdap.api.dataset.table.Table) HashMap(java.util.HashMap) TransactionExecutor(org.apache.tephra.TransactionExecutor) DatasetId(co.cask.cdap.proto.id.DatasetId) Predicate(com.google.common.base.Predicate) HashMap(java.util.HashMap) Map(java.util.Map) Test(org.junit.Test)

Example 67 with TransactionExecutor

use of org.apache.tephra.TransactionExecutor in project cdap by caskdata.

the class MetadataStoreDatasetTest method testScan.

@Test
public void testScan() throws Exception {
    DatasetId storeTable = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("testScan");
    dsFrameworkUtil.createInstance(Table.class.getName(), storeTable, DatasetProperties.EMPTY);
    Table table = dsFrameworkUtil.getInstance(storeTable);
    Assert.assertNotNull(table);
    final MetadataStoreDataset metadataStoreDataset = new MetadataStoreDataset(table);
    TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor((TransactionAware) table);
    // Write some values
    final List<Integer> expected = new ArrayList<>();
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            for (int i = 0; i < 25; ++i) {
                MDSKey mdsKey = new MDSKey.Builder().add(i).build();
                metadataStoreDataset.write(mdsKey, i);
                expected.add(i);
            }
        }
    });
    MDSKey start = new MDSKey.Builder().add(0).build();
    final MDSKey end = new MDSKey.Builder().add(25).build();
    List<Integer> actual = new ArrayList<>();
    // Use scan limit of 3
    int scanLimit = 3;
    int runs = 0;
    while (true) {
        final ScanFunction function = new ScanFunction(scanLimit);
        final MDSKey finalStart = start;
        txnl.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                metadataStoreDataset.scan(finalStart, end, Integer.class, function);
            }
        });
        if (function.getNumProcessed() == 0) {
            break;
        }
        ++runs;
        Assert.assertTrue(function.getValues().size() <= scanLimit);
        actual.addAll(function.getValues());
        start = new MDSKey(Bytes.stopKeyForPrefix(function.getLastKey().getKey()));
    }
    Assert.assertEquals(9, runs);
    Assert.assertEquals(expected, actual);
}
Also used : Table(co.cask.cdap.api.dataset.table.Table) ArrayList(java.util.ArrayList) TransactionExecutor(org.apache.tephra.TransactionExecutor) DatasetId(co.cask.cdap.proto.id.DatasetId) Test(org.junit.Test)

Example 68 with TransactionExecutor

use of org.apache.tephra.TransactionExecutor in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testSimpleDataset.

@Test
public void testSimpleDataset() throws Exception {
    // Configuring Dataset types
    DatasetFramework framework = getFramework();
    // system namespace has a module orderedTable-inMemory
    Assert.assertTrue(framework.hasSystemType("table"));
    // myspace namespace has no modules
    Assert.assertFalse(framework.hasType(IN_MEMORY_TYPE));
    Assert.assertFalse(framework.hasType(SIMPLE_KV_TYPE));
    // add module to namespace 'myspace'
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    // make sure it got added to 'myspace'
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    // but not to 'system'
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    // Creating instance using a type from own namespace
    framework.addInstance(SimpleKVTable.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    // verify it got added to the right namespace
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    // and not to the system namespace
    Assert.assertFalse(framework.hasInstance(NamespaceId.SYSTEM.dataset("my_table")));
    // Doing some admin and data ops
    DatasetAdmin admin = framework.getAdmin(MY_TABLE, null);
    Assert.assertNotNull(admin);
    final SimpleKVTable kvTable = framework.getDataset(MY_TABLE, DatasetDefinition.NO_ARGUMENTS, null);
    Assert.assertNotNull(kvTable);
    TransactionExecutor txnl = new DefaultTransactionExecutor(new MinimalTxSystemClient(), (TransactionAware) kvTable);
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            kvTable.put("key1", "value1");
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals("value1", kvTable.get("key1"));
        }
    });
    admin.truncate();
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertTrue(kvTable.get("key1") == null);
        }
    });
    // cleanup
    framework.deleteInstance(MY_TABLE);
    framework.deleteModule(KEY_VALUE);
    // recreate instance without adding a module in 'myspace'. This should use types from default namespace
    framework.addInstance("table", MY_TABLE, DatasetProperties.EMPTY);
    // verify it got added to the right namespace
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    admin = framework.getAdmin(MY_TABLE, null);
    Assert.assertNotNull(admin);
    final Table table = framework.getDataset(MY_TABLE, DatasetDefinition.NO_ARGUMENTS, null);
    Assert.assertNotNull(table);
    txnl = new DefaultTransactionExecutor(new MinimalTxSystemClient(), (TransactionAware) table);
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            table.put(Bytes.toBytes("key1"), Bytes.toBytes("column1"), Bytes.toBytes("value1"));
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals("value1", Bytes.toString(table.get(Bytes.toBytes("key1"), Bytes.toBytes("column1"))));
        }
    });
    // cleanup
    framework.deleteInstance(MY_TABLE);
}
Also used : LineageWriterDatasetFramework(co.cask.cdap.data2.metadata.writer.LineageWriterDatasetFramework) Table(co.cask.cdap.api.dataset.table.Table) TransactionAware(org.apache.tephra.TransactionAware) DefaultTransactionExecutor(org.apache.tephra.DefaultTransactionExecutor) DatasetAdmin(co.cask.cdap.api.dataset.DatasetAdmin) TransactionExecutor(org.apache.tephra.TransactionExecutor) DefaultTransactionExecutor(org.apache.tephra.DefaultTransactionExecutor) InstanceConflictException(co.cask.cdap.api.dataset.InstanceConflictException) DatasetManagementException(co.cask.cdap.api.dataset.DatasetManagementException) IOException(java.io.IOException) MinimalTxSystemClient(org.apache.tephra.inmemory.MinimalTxSystemClient) Test(org.junit.Test)

Example 69 with TransactionExecutor

use of org.apache.tephra.TransactionExecutor in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testCompositeDataset.

private void testCompositeDataset(DatasetFramework framework) throws Exception {
    // Doing some admin and data ops
    DatasetAdmin admin = framework.getAdmin(MY_TABLE, null);
    Assert.assertNotNull(admin);
    final KeyValueTable table = framework.getDataset(MY_TABLE, DatasetDefinition.NO_ARGUMENTS, null);
    Assert.assertNotNull(table);
    TransactionExecutor txnl = new DefaultTransactionExecutor(new MinimalTxSystemClient(), (TransactionAware) table);
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            table.put("key1", "value1");
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals("value1", table.get("key1"));
        }
    });
    admin.truncate();
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals(null, table.get("key1"));
        }
    });
}
Also used : DefaultTransactionExecutor(org.apache.tephra.DefaultTransactionExecutor) DatasetAdmin(co.cask.cdap.api.dataset.DatasetAdmin) TransactionExecutor(org.apache.tephra.TransactionExecutor) DefaultTransactionExecutor(org.apache.tephra.DefaultTransactionExecutor) InstanceConflictException(co.cask.cdap.api.dataset.InstanceConflictException) DatasetManagementException(co.cask.cdap.api.dataset.DatasetManagementException) IOException(java.io.IOException) MinimalTxSystemClient(org.apache.tephra.inmemory.MinimalTxSystemClient)

Example 70 with TransactionExecutor

use of org.apache.tephra.TransactionExecutor in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testNamespaceInstanceIsolation.

@Test
@SuppressWarnings("ConstantConditions")
public void testNamespaceInstanceIsolation() throws Exception {
    DatasetFramework framework = getFramework();
    // create 2 namespaces
    NamespaceId namespace1 = new NamespaceId("ns1");
    NamespaceId namespace2 = new NamespaceId("ns2");
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(namespace1).build());
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(namespace2).build());
    namespacedLocationFactory.get(namespace1).mkdirs();
    namespacedLocationFactory.get(namespace2).mkdirs();
    // create 2 tables, one in each namespace. both tables have the same name.
    DatasetId table1ID = namespace1.dataset("table");
    DatasetId table2ID = namespace2.dataset("table");
    // have slightly different properties so that we can distinguish between them
    framework.addInstance(Table.class.getName(), table1ID, DatasetProperties.builder().add("tag", "table1").build());
    framework.addInstance(Table.class.getName(), table2ID, DatasetProperties.builder().add("tag", "table2").build());
    // perform some data operations to make sure they are not the same underlying table
    final Table table1 = framework.getDataset(table1ID, Maps.<String, String>newHashMap(), null);
    final Table table2 = framework.getDataset(table2ID, Maps.<String, String>newHashMap(), null);
    TransactionExecutor txnl = new DefaultTransactionExecutor(new MinimalTxSystemClient(), (TransactionAware) table1, (TransactionAware) table2);
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            table1.put(Bytes.toBytes("rowkey"), Bytes.toBytes("column"), Bytes.toBytes("val1"));
            table2.put(Bytes.toBytes("rowkey"), Bytes.toBytes("column"), Bytes.toBytes("val2"));
        }
    });
    // check data is different, which means they are different underlying tables
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals("val1", Bytes.toString(table1.get(Bytes.toBytes("rowkey"), Bytes.toBytes("column"))));
            Assert.assertEquals("val2", Bytes.toString(table2.get(Bytes.toBytes("rowkey"), Bytes.toBytes("column"))));
        }
    });
    // check get all in a namespace only includes those in that namespace
    Collection<DatasetSpecificationSummary> specs = framework.getInstances(namespace1);
    Assert.assertEquals(1, specs.size());
    Assert.assertEquals("table1", specs.iterator().next().getProperties().get("tag"));
    specs = framework.getInstances(namespace2);
    Assert.assertEquals(1, specs.size());
    Assert.assertEquals("table2", specs.iterator().next().getProperties().get("tag"));
    // delete one instance and make sure the other still exists
    framework.deleteInstance(table1ID);
    Assert.assertFalse(framework.hasInstance(table1ID));
    Assert.assertTrue(framework.hasInstance(table2ID));
    // delete all instances in one namespace and make sure the other still exists
    framework.addInstance(Table.class.getName(), table1ID, DatasetProperties.EMPTY);
    framework.deleteAllInstances(namespace1);
    Assert.assertTrue(framework.hasInstance(table2ID));
    // delete one namespace and make sure the other still exists
    namespacedLocationFactory.get(namespace1).delete(true);
    Assert.assertTrue(framework.hasInstance(table2ID));
}
Also used : Table(co.cask.cdap.api.dataset.table.Table) TransactionExecutor(org.apache.tephra.TransactionExecutor) DefaultTransactionExecutor(org.apache.tephra.DefaultTransactionExecutor) DatasetSpecificationSummary(co.cask.cdap.proto.DatasetSpecificationSummary) InstanceConflictException(co.cask.cdap.api.dataset.InstanceConflictException) DatasetManagementException(co.cask.cdap.api.dataset.DatasetManagementException) IOException(java.io.IOException) DatasetId(co.cask.cdap.proto.id.DatasetId) MinimalTxSystemClient(org.apache.tephra.inmemory.MinimalTxSystemClient) LineageWriterDatasetFramework(co.cask.cdap.data2.metadata.writer.LineageWriterDatasetFramework) NamespaceMeta(co.cask.cdap.proto.NamespaceMeta) DefaultTransactionExecutor(org.apache.tephra.DefaultTransactionExecutor) NamespaceId(co.cask.cdap.proto.id.NamespaceId) Test(org.junit.Test)

Aggregations

TransactionExecutor (org.apache.tephra.TransactionExecutor)79 Test (org.junit.Test)64 TransactionFailureException (org.apache.tephra.TransactionFailureException)31 DatasetId (co.cask.cdap.proto.id.DatasetId)30 Table (co.cask.cdap.api.dataset.table.Table)15 List (java.util.List)15 NoSuchElementException (java.util.NoSuchElementException)15 TransactionAware (org.apache.tephra.TransactionAware)15 IOException (java.io.IOException)12 Row (co.cask.cdap.api.dataset.table.Row)11 DefaultTransactionExecutor (org.apache.tephra.DefaultTransactionExecutor)11 Put (co.cask.cdap.api.dataset.table.Put)10 ImmutableList (com.google.common.collect.ImmutableList)9 ProgramId (co.cask.cdap.proto.id.ProgramId)8 DatasetManagementException (co.cask.cdap.api.dataset.DatasetManagementException)6 Scanner (co.cask.cdap.api.dataset.table.Scanner)6 Map (java.util.Map)6 ProgramRunId (co.cask.cdap.proto.id.ProgramRunId)5 TypeToken (com.google.common.reflect.TypeToken)5 Iterator (java.util.Iterator)5