Examples with TransactionAware org.apache.tephra.TransactionAware used on opensource projects

Example 66 with TransactionAware

use of org.apache.tephra.TransactionAware in project cdap by caskdata.

the class TimePartitionedFileSetTest method validateInputPaths.

/**
   * Validates that the output configuration of the tpfs, when instantiated with (time - start * minutes) as
   * input start time and (time + end * minutes) as input end time, returns the expected list of paths.
   */
private void validateInputPaths(long time, long start, long end, final String... expected) throws IOException, DatasetManagementException, InterruptedException, TransactionFailureException {
    Map<String, String> arguments = Maps.newHashMap();
    TimePartitionedFileSetArguments.setInputStartTime(arguments, time + start * MINUTE);
    TimePartitionedFileSetArguments.setInputEndTime(arguments, time + end * MINUTE);
    final TimePartitionedFileSet tpfs = dsFrameworkUtil.getInstance(TPFS_INSTANCE, arguments);
    TransactionAware txAwareDataset = (TransactionAware) tpfs;
    dsFrameworkUtil.newInMemoryTransactionExecutor(txAwareDataset).execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Map<String, String> inputConfig = tpfs.getInputFormatConfiguration();
            String inputs = inputConfig.get(FileInputFormat.INPUT_DIR);
            Assert.assertNotNull(inputs);
            if (expected.length == 0) {
                Assert.assertTrue(inputs.isEmpty());
                return;
            }
            String[] inputPaths = inputs.split(",");
            Assert.assertEquals(expected.length, inputPaths.length);
            // order is not guaranteed.
            Arrays.sort(expected);
            Arrays.sort(inputPaths);
            for (int i = 0; i < expected.length; i++) {
                // every input path is absolute, whereas expected paths are relative
                Assert.assertTrue("path #" + i + " does not match", inputPaths[i].endsWith(expected[i]));
            }
        }
    });
}

Example 67 with TransactionAware

use of org.apache.tephra.TransactionAware in project cdap by caskdata.

the class TimePartitionedFileSetTest method testPartitionMetadata.

@Test
public void testPartitionMetadata() throws Exception {
    final TimePartitionedFileSet tpfs = dsFrameworkUtil.getInstance(TPFS_INSTANCE);
    TransactionAware txAware = (TransactionAware) tpfs;
    dsFrameworkUtil.newInMemoryTransactionExecutor(txAware).execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // make sure the dataset has no partitions
            validateTimePartitions(tpfs, 0L, MAX, Collections.<Long, String>emptyMap());
            Date date = DATE_FORMAT.parse("6/4/12 10:00 am");
            long time = date.getTime();
            // keep track of all the metadata added
            Map<String, String> allMetadata = Maps.newHashMap();
            Map<String, String> metadata = ImmutableMap.of("key1", "value1", "key2", "value3", "key100", "value4");
            tpfs.addPartition(time, "file", metadata);
            allMetadata.putAll(metadata);
            TimePartitionDetail partitionByTime = tpfs.getPartitionByTime(time);
            Assert.assertNotNull(partitionByTime);
            Assert.assertEquals(metadata, partitionByTime.getMetadata().asMap());
            tpfs.addMetadata(time, "key3", "value4");
            allMetadata.put("key3", "value4");
            try {
                // attempting to update an existing key throws a DatasetException
                tpfs.addMetadata(time, "key3", "value5");
                Assert.fail("Expected not to be able to update an existing metadata entry");
            } catch (DataSetException expected) {
            }
            Map<String, String> newMetadata = ImmutableMap.of("key4", "value4", "key5", "value5");
            tpfs.addMetadata(time, newMetadata);
            allMetadata.putAll(newMetadata);
            partitionByTime = tpfs.getPartitionByTime(time);
            Assert.assertNotNull(partitionByTime);
            Assert.assertEquals(allMetadata, partitionByTime.getMetadata().asMap());
        }
    });
}

Example 68 with TransactionAware

use of org.apache.tephra.TransactionAware in project cdap by caskdata.

the class TimePartitionedFileSetTest method testInputConfiguration.

private void testInputConfiguration(Map<String, String> arguments, final String expectedPath) throws Exception {
    final TimePartitionedFileSet dataset = dsFrameworkUtil.getInstance(TPFS_INSTANCE, arguments);
    TransactionAware txAwareDataset = (TransactionAware) dataset;
    dsFrameworkUtil.newInMemoryTransactionExecutor(txAwareDataset).execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Map<String, String> inputConf = dataset.getInputFormatConfiguration();
            String input = inputConf.get(FileInputFormat.INPUT_DIR);
            Assert.assertNotNull(input);
            String[] inputs = input.split(",");
            Assert.assertEquals(1, inputs.length);
            Assert.assertTrue(inputs[0].endsWith(expectedPath));
        }
    });
}

Example 69 with TransactionAware

use of org.apache.tephra.TransactionAware in project cdap by caskdata.

the class TableConcurrentTest method testConcurrentOnSingleTable.

@Test(timeout = 120000)
public void testConcurrentOnSingleTable() throws Exception {
    // Set of clients read and write data concurrently.
    // * n clients increment a value with increasing values (+1, +2, ...) at specific row:column 100 times
    // * n clients append 100 columns to a set of 4 rows which includes the row that gets incremented (2 at a time).
    //   Append is: read all columns, add <last_column+1>
    // todo: improve to use deletes. E.g. in append - remove all existing before appending new
    final int n = 5;
    getTableAdmin(CONTEXT1, MY_TABLE).create();
    try {
        ExecutorService executor = Executors.newFixedThreadPool(n * 2);
        // start threads
        for (int i = 0; i < n; i++) {
            executor.submit(new IncrementingClient(txExecutorFactory));
            executor.submit(new AppendingClient(txExecutorFactory));
        }
        // wait for finish
        executor.shutdown();
        executor.awaitTermination(2, TimeUnit.MINUTES);
        // verify result
        final T table = getTable(CONTEXT1, MY_TABLE);
        TransactionExecutor txExecutor = txExecutorFactory.createExecutor(Lists.newArrayList((TransactionAware) table));
        txExecutor.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                verifyIncrements();
                verifyAppends();
            }

            private void verifyAppends() throws Exception {
                for (byte[] row : ROWS_TO_APPEND_TO) {
                    Map<byte[], byte[]> cols = table.get(row).getColumns();
                    Assert.assertFalse(cols.isEmpty());
                    // +1 because there was one extra column that we incremented
                    boolean isIncrementedColumn = Arrays.equals(ROW_TO_INCREMENT, row);
                    Assert.assertEquals(n * 100 + (isIncrementedColumn ? 1 : 0), cols.size());
                    for (int i = 0; i < n * 100; i++) {
                        Assert.assertArrayEquals(Bytes.toBytes("foo" + i), cols.get(Bytes.toBytes("column" + i)));
                    }
                }
            }

            private void verifyIncrements() throws Exception {
                Map<byte[], byte[]> result = table.get(ROW_TO_INCREMENT, new byte[][] { COLUMN_TO_INCREMENT }).getColumns();
                Assert.assertFalse(result.isEmpty());
                byte[] val = result.get(COLUMN_TO_INCREMENT);
                long sum1to100 = ((1 + 99) * 99 / 2);
                Assert.assertEquals(n * sum1to100, Bytes.toLong(val));
            }
        });
    } finally {
        getTableAdmin(CONTEXT1, MY_TABLE).drop();
    }
}

Example 70 with TransactionAware

use of org.apache.tephra.TransactionAware in project cdap by caskdata.

the class TableTest method testClientSurvivesTableReset.

// this test ensures that an existing client survives the truncating or dropping and recreating of a table
@Test
public void testClientSurvivesTableReset() throws Exception {
    final String tableName = "survive";
    DatasetAdmin admin = getTableAdmin(CONTEXT1, tableName);
    admin.create();
    Table table = getTable(CONTEXT1, tableName);
    // write some values
    Transaction tx0 = txClient.startShort();
    ((TransactionAware) table).startTx(tx0);
    table.put(R1, a(C1), a(V1));
    Assert.assertTrue(txClient.canCommit(tx0, ((TransactionAware) table).getTxChanges()));
    Assert.assertTrue(((TransactionAware) table).commitTx());
    Assert.assertTrue(txClient.commit(tx0));
    ((TransactionAware) table).postTxCommit();
    // TableAssert.verify
    Transaction tx1 = txClient.startShort();
    ((TransactionAware) table).startTx(tx1);
    TableAssert.assertRow(a(C1, V1), table.get(R1));
    // drop table and recreate
    admin.drop();
    admin.create();
    // TableAssert.verify can read but nothing there
    TableAssert.assertRow(a(), table.get(R1));
    // only did read, safe to abort
    txClient.abort(tx1);
    // create a new client and write another value
    Table table2 = getTable(CONTEXT1, tableName);
    Transaction tx2 = txClient.startShort();
    ((TransactionAware) table2).startTx(tx2);
    table2.put(R1, a(C2), a(V2));
    Assert.assertTrue(txClient.canCommit(tx2, ((TransactionAware) table2).getTxChanges()));
    Assert.assertTrue(((TransactionAware) table2).commitTx());
    Assert.assertTrue(txClient.commit(tx2));
    ((TransactionAware) table2).postTxCommit();
    // TableAssert.verify it is visible
    Transaction tx3 = txClient.startShort();
    ((TransactionAware) table).startTx(tx3);
    TableAssert.assertRow(a(C2, V2), table.get(R1));
    // truncate table
    admin.truncate();
    // TableAssert.verify can read but nothing there
    TableAssert.assertRow(a(), table.get(R1));
    // only did read, safe to abort
    txClient.abort(tx3);
    // write again with other client
    Transaction tx4 = txClient.startShort();
    ((TransactionAware) table2).startTx(tx4);
    table2.put(R1, a(C3), a(V3));
    Assert.assertTrue(txClient.canCommit(tx4, ((TransactionAware) table2).getTxChanges()));
    Assert.assertTrue(((TransactionAware) table2).commitTx());
    Assert.assertTrue(txClient.commit(tx4));
    ((TransactionAware) table2).postTxCommit();
    // TableAssert.verify it is visible
    Transaction tx5 = txClient.startShort();
    ((TransactionAware) table).startTx(tx5);
    TableAssert.assertRow(a(C3, V3), table.get(R1));
    // only did read, safe to abort
    txClient.abort(tx5);
    // drop table
    admin.drop();
}