Examples with TransactionFailureException org.apache.tephra.TransactionFailureException used on opensource projects

Example 51 with TransactionFailureException

use of org.apache.tephra.TransactionFailureException in project cdap by caskdata.

the class TimePartitionedFileSetTest method validateInputPaths.

/**
   * Validates that the output configuration of the tpfs, when instantiated with (time - start * minutes) as
   * input start time and (time + end * minutes) as input end time, returns the expected list of paths.
   */
private void validateInputPaths(long time, long start, long end, final String... expected) throws IOException, DatasetManagementException, InterruptedException, TransactionFailureException {
    Map<String, String> arguments = Maps.newHashMap();
    TimePartitionedFileSetArguments.setInputStartTime(arguments, time + start * MINUTE);
    TimePartitionedFileSetArguments.setInputEndTime(arguments, time + end * MINUTE);
    final TimePartitionedFileSet tpfs = dsFrameworkUtil.getInstance(TPFS_INSTANCE, arguments);
    TransactionAware txAwareDataset = (TransactionAware) tpfs;
    dsFrameworkUtil.newInMemoryTransactionExecutor(txAwareDataset).execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Map<String, String> inputConfig = tpfs.getInputFormatConfiguration();
            String inputs = inputConfig.get(FileInputFormat.INPUT_DIR);
            Assert.assertNotNull(inputs);
            if (expected.length == 0) {
                Assert.assertTrue(inputs.isEmpty());
                return;
            }
            String[] inputPaths = inputs.split(",");
            Assert.assertEquals(expected.length, inputPaths.length);
            // order is not guaranteed.
            Arrays.sort(expected);
            Arrays.sort(inputPaths);
            for (int i = 0; i < expected.length; i++) {
                // every input path is absolute, whereas expected paths are relative
                Assert.assertTrue("path #" + i + " does not match", inputPaths[i].endsWith(expected[i]));
            }
        }
    });
}

Example 52 with TransactionFailureException

use of org.apache.tephra.TransactionFailureException in project cdap by caskdata.

the class PartitionedFileSetDataset method fixPartitions.

/**
   * This method can bring a partitioned file set in sync with explore. It scans the partition table and adds
   * every partition to explore. It will start multiple transactions, processing a batch of partitions in each
   * transaction. Optionally, it can disable and re-enable explore first, that is, drop and recreate the Hive table.
   * @param transactional the Transactional for executing transactions
   * @param datasetName the name of the dataset to fix
   * @param doDisable whether to disable and re-enable explore first
   * @param partitionsPerTx how many partitions to process per transaction
   * @param verbose whether to log verbosely. If true, this will log a message for every partition; otherwise it
   *                will only log a report of how many partitions were added / could not be added.
   */
@Beta
@SuppressWarnings("unused")
public static void fixPartitions(Transactional transactional, final String datasetName, boolean doDisable, final int partitionsPerTx, final boolean verbose) {
    if (doDisable) {
        try {
            transactional.execute(new TxRunnable() {

                @Override
                public void run(co.cask.cdap.api.data.DatasetContext context) throws Exception {
                    PartitionedFileSetDataset pfs = context.getDataset(datasetName);
                    pfs.disableExplore();
                    // truncating = true, because this is like truncating
                    pfs.enableExplore(true);
                }
            });
        } catch (TransactionFailureException e) {
            throw new DataSetException("Unable to disable and enable Explore", e.getCause());
        } catch (RuntimeException e) {
            if (e.getCause() instanceof TransactionFailureException) {
                throw new DataSetException("Unable to disable and enable Explore", e.getCause().getCause());
            }
            throw e;
        }
    }
    final AtomicReference<PartitionKey> startKey = new AtomicReference<>();
    final AtomicLong errorCount = new AtomicLong(0L);
    final AtomicLong successCount = new AtomicLong(0L);
    do {
        try {
            transactional.execute(new TxRunnable() {

                @Override
                public void run(co.cask.cdap.api.data.DatasetContext context) throws Exception {
                    final PartitionedFileSetDataset pfs = context.getDataset(datasetName);
                    // compute start row for the scan, reset remembered start key to null
                    byte[] startRow = startKey.get() == null ? null : generateRowKey(startKey.get(), pfs.getPartitioning());
                    startKey.set(null);
                    PartitionConsumer consumer = new PartitionConsumer() {

                        int count = 0;

                        @Override
                        public void consume(PartitionKey key, String path, @Nullable PartitionMetadata metadata) {
                            if (count >= partitionsPerTx) {
                                // reached the limit: remember this key as the start for the next round
                                startKey.set(key);
                                return;
                            }
                            try {
                                pfs.addPartitionToExplore(key, path);
                                successCount.incrementAndGet();
                                if (verbose) {
                                    LOG.info("Added partition {} with path {}", key, path);
                                }
                            } catch (DataSetException e) {
                                errorCount.incrementAndGet();
                                if (verbose) {
                                    LOG.warn(e.getMessage(), e);
                                }
                            }
                            count++;
                        }
                    };
                    pfs.getPartitions(null, consumer, false, startRow, null, partitionsPerTx + 1);
                }
            });
        } catch (TransactionConflictException e) {
            throw new DataSetException("Transaction conflict while reading partitions. This should never happen. " + "Make sure that no other programs are using this dataset at the same time.");
        } catch (TransactionFailureException e) {
            throw new DataSetException("Transaction failure: " + e.getMessage(), e.getCause());
        } catch (RuntimeException e) {
            // this looks like duplication but is needed in case this is run from a worker: see CDAP-6837
            if (e.getCause() instanceof TransactionConflictException) {
                throw new DataSetException("Transaction conflict while reading partitions. This should never happen. " + "Make sure that no other programs are using this dataset at the same time.");
            } else if (e.getCause() instanceof TransactionFailureException) {
                throw new DataSetException("Transaction failure: " + e.getMessage(), e.getCause().getCause());
            } else {
                throw e;
            }
        }
    } while (// if it is null, then we consumed less than the limit in this round -> done
    startKey.get() != null);
    LOG.info("Added {} partitions, failed to add {} partitions.", successCount.get(), errorCount.get());
}

Example 53 with TransactionFailureException

use of org.apache.tephra.TransactionFailureException in project cdap by caskdata.

the class AppWithCustomTx method attemptNestedTransaction.

/**
   * Attempt to nest transactions. we expect this to fail, and if it does, we write the value "failed"
   * to the table, for the test case to validate.
   */
static void attemptNestedTransaction(Transactional txnl, final String row, final String key) {
    try {
        txnl.execute(new TxRunnable() {

            @Override
            public void run(DatasetContext ctext) throws Exception {
                recordTransaction(ctext, row, key);
            }
        });
        LOG.error("Nested transaction should not have succeeded for {}:{}", row, key);
    } catch (TransactionFailureException e) {
        // expected: starting nested transaction should fail
        LOG.info("Nested transaction failed as expected for {}:{}", row, key);
    } catch (RuntimeException e) {
        // TODO (CDAP-6837): this is needed because worker's execute() propagates the tx failure as a runtime exception
        if (e.getCause() instanceof TransactionFailureException) {
            // expected: starting nested transaction should fail
            LOG.info("Nested transaction failed as expected for {}:{}", row, key);
        } else {
            throw e;
        }
    }
    // we know that the transactional is a program context and hence implement DatasetContext
    TransactionCapturingTable capture = ((DatasetContext) txnl).getDataset(CAPTURE);
    capture.getTable().put(new Put(row, key, FAILED));
}

Example 54 with TransactionFailureException

use of org.apache.tephra.TransactionFailureException in project cdap by caskdata.

the class SparkTransactionHandlerTest method testFailureTransaction.

/**
   * Tests the case where starting of transaction failed.
   */
@Test
public void testFailureTransaction() throws Exception {
    TransactionManager txManager = new TransactionManager(new Configuration()) {

        @Override
        public Transaction startLong() {
            throw new IllegalStateException("Cannot start long transaction");
        }
    };
    txManager.startAndWait();
    try {
        SparkTransactionHandler txHandler = new SparkTransactionHandler(new InMemoryTxSystemClient(txManager));
        SparkDriverHttpService httpService = new SparkDriverHttpService("test", InetAddress.getLoopbackAddress().getCanonicalHostName(), txHandler);
        httpService.startAndWait();
        try {
            // Start a job
            txHandler.jobStarted(1, ImmutableSet.of(2));
            // Make a call to the stage transaction endpoint, it should throw TransactionFailureException
            try {
                new SparkTransactionClient(httpService.getBaseURI()).getTransaction(2, 1, TimeUnit.SECONDS);
                Assert.fail("Should failed to get transaction");
            } catch (TransactionFailureException e) {
            // expected
            }
            // End the job
            txHandler.jobEnded(1, false);
        } finally {
            httpService.stopAndWait();
        }
    } finally {
        txManager.stopAndWait();
    }
}