Examples with TransactionConflictException org.apache.tephra.TransactionConflictException used on opensource projects

Example 6 with TransactionConflictException

use of org.apache.tephra.TransactionConflictException in project cdap by caskdata.

the class DefaultStore method upgrade.

/**
   * Method to add version in DefaultStore.
   *
   * @throws InterruptedException
   * @throws IOException
   * @throws DatasetManagementException
   */
public void upgrade() throws InterruptedException, IOException, DatasetManagementException {
    // If upgrade is already complete, then simply return.
    if (isUpgradeComplete()) {
        LOG.info("{} is already upgraded.", NAME);
        return;
    }
    final AtomicInteger maxRows = new AtomicInteger(1000);
    final AtomicInteger sleepTimeInSecs = new AtomicInteger(60);
    LOG.info("Starting upgrade of {}.", NAME);
    // to check whether they need to do additional scans to accommodate old data formats.
    while (!isUpgradeComplete()) {
        sleepTimeInSecs.set(60);
        try {
            Transactions.execute(transactional, new TxCallable<Void>() {

                @Override
                public Void call(DatasetContext context) throws Exception {
                    AppMetadataStore store = getAppMetadataStore(context);
                    boolean upgradeComplete = store.upgradeVersionKeys(maxRows.get());
                    if (upgradeComplete) {
                        store.setUpgradeComplete(APP_VERSION_UPGRADE_KEY);
                    }
                    return null;
                }
            });
        } catch (TransactionFailureException e) {
            if (e instanceof TransactionConflictException) {
                LOG.debug("Upgrade step faced Transaction Conflict exception. Retrying operation now.", e);
                sleepTimeInSecs.set(10);
            } else if (e instanceof TransactionNotInProgressException) {
                int currMaxRows = maxRows.get();
                if (currMaxRows > 500) {
                    maxRows.decrementAndGet();
                } else {
                    LOG.warn("Could not complete upgrade of {}, tried for 500 times", NAME);
                    return;
                }
                sleepTimeInSecs.set(10);
                LOG.debug("Upgrade step faced a Transaction Timeout exception. " + "Reducing the number of max rows to : {} and retrying the operation now.", maxRows.get(), e);
            } else {
                LOG.error("Upgrade step faced exception. Will retry operation after some delay.", e);
                sleepTimeInSecs.set(60);
            }
        }
        TimeUnit.SECONDS.sleep(sleepTimeInSecs.get());
    }
    LOG.info("Upgrade of {} is complete.", NAME);
}

Example 7 with TransactionConflictException

use of org.apache.tephra.TransactionConflictException in project cdap by caskdata.

the class ArtifactStore method write.

/**
   * Write the artifact and its metadata to the store. Once added, artifacts cannot be changed unless they are
   * snapshot versions.
   *
   * @param artifactId the id of the artifact to add
   * @param artifactMeta the metadata for the artifact
   * @param artifactContentSupplier the supplier for the input stream of the contents of the artifact
   * @return detail about the newly added artifact
   * @throws WriteConflictException if the artifact is already currently being written
   * @throws ArtifactAlreadyExistsException if a non-snapshot version of the artifact already exists
   * @throws IOException if there was an exception persisting the artifact contents to the filesystem,
   *                     of persisting the artifact metadata to the metastore
   */
public ArtifactDetail write(final Id.Artifact artifactId, final ArtifactMeta artifactMeta, final InputSupplier<? extends InputStream> artifactContentSupplier, EntityImpersonator entityImpersonator) throws WriteConflictException, ArtifactAlreadyExistsException, IOException {
    // if we're not a snapshot version, check that the artifact doesn't exist already.
    final ArtifactCell artifactCell = new ArtifactCell(artifactId);
    if (!artifactId.getVersion().isSnapshot()) {
        try {
            transactional.execute(new TxRunnable() {

                @Override
                public void run(DatasetContext context) throws Exception {
                    if (getMetaTable(context).get(artifactCell.rowkey, artifactCell.column) != null) {
                        throw new ArtifactAlreadyExistsException(artifactId.toEntityId());
                    }
                }
            });
        } catch (TransactionFailureException e) {
            throw Transactions.propagate(e, ArtifactAlreadyExistsException.class, IOException.class);
        }
    }
    final Location destination;
    try {
        destination = copyFileToDestination(artifactId, artifactContentSupplier, entityImpersonator);
    } catch (Exception e) {
        Throwables.propagateIfInstanceOf(e, IOException.class);
        throw Throwables.propagate(e);
    }
    // now try and write the metadata for the artifact
    try {
        transactional.execute(new TxRunnable() {

            @Override
            public void run(DatasetContext context) throws Exception {
                // we have to check that the metadata doesn't exist again since somebody else may have written
                // the artifact while we were copying the artifact to the filesystem.
                Table metaTable = getMetaTable(context);
                byte[] existingMetaBytes = metaTable.get(artifactCell.rowkey, artifactCell.column);
                boolean isSnapshot = artifactId.getVersion().isSnapshot();
                if (existingMetaBytes != null && !isSnapshot) {
                    // non-snapshot artifacts are immutable. If there is existing metadata, stop here.
                    throw new ArtifactAlreadyExistsException(artifactId.toEntityId());
                }
                ArtifactData data = new ArtifactData(destination, artifactMeta);
                // this means cleaning up the old jar, and deleting plugin and app rows.
                if (existingMetaBytes != null) {
                    deleteMeta(metaTable, artifactId, existingMetaBytes);
                }
                // write artifact metadata
                writeMeta(metaTable, artifactId, data);
            }
        });
        return new ArtifactDetail(new ArtifactDescriptor(artifactId.toArtifactId(), destination), artifactMeta);
    } catch (TransactionConflictException e) {
        destination.delete();
        throw new WriteConflictException(artifactId);
    } catch (TransactionFailureException e) {
        destination.delete();
        throw Transactions.propagate(e, ArtifactAlreadyExistsException.class, IOException.class);
    }
}

Example 8 with TransactionConflictException

use of org.apache.tephra.TransactionConflictException in project cdap by caskdata.

the class DatasetBasedStreamSizeScheduleStore method upgrade.

/**
   * Method to add version in StreamSizeSchedule row key in SchedulerStore.
   *
   * @throws InterruptedException
   * @throws IOException
   * @throws DatasetManagementException
   */
public void upgrade() throws InterruptedException, IOException, DatasetManagementException {
    // Wait until the store is initialized
    // Use a new instance of table since Table is not thread safe
    Table metaTable = null;
    while (metaTable == null) {
        try {
            metaTable = tableUtil.getMetaTable();
        } catch (Exception e) {
        // ignore exception
        }
        TimeUnit.SECONDS.sleep(10);
    }
    if (isUpgradeComplete()) {
        LOG.info("{} is already upgraded.", NAME);
        return;
    }
    final AtomicInteger maxNumberUpdateRows = new AtomicInteger(1000);
    final AtomicInteger sleepTimeInSecs = new AtomicInteger(60);
    LOG.info("Starting upgrade of {}.", NAME);
    while (true) {
        sleepTimeInSecs.set(60);
        try {
            if (executeUpgradeInTransaction(table, maxNumberUpdateRows)) {
                break;
            }
        } catch (TransactionFailureException e) {
            if (e instanceof TransactionConflictException) {
                LOG.debug("Upgrade step faced Transaction Conflict exception. Retrying operation now.", e);
                sleepTimeInSecs.set(10);
            } else if (e instanceof TransactionNotInProgressException) {
                int currMaxRows = maxNumberUpdateRows.get();
                if (currMaxRows > 500) {
                    maxNumberUpdateRows.decrementAndGet();
                } else {
                    LOG.warn("Could not complete upgrade of {}, tried for 500 times", NAME);
                    return;
                }
                sleepTimeInSecs.set(10);
                LOG.debug("Upgrade step faced a Transaction Timeout exception. " + "Current number of max update rows is set to : {} and retrying the operation now.", maxNumberUpdateRows.get(), e);
            } else {
                LOG.error("Upgrade step faced exception. Will retry operation after some delay.", e);
                sleepTimeInSecs.set(60);
            }
        }
        TimeUnit.SECONDS.sleep(sleepTimeInSecs.get());
    }
    LOG.info("Upgrade of {} is complete.", NAME);
}

Example 9 with TransactionConflictException

use of org.apache.tephra.TransactionConflictException in project cdap by caskdata.

the class PartitionedFileSetDataset method fixPartitions.

/**
   * This method can bring a partitioned file set in sync with explore. It scans the partition table and adds
   * every partition to explore. It will start multiple transactions, processing a batch of partitions in each
   * transaction. Optionally, it can disable and re-enable explore first, that is, drop and recreate the Hive table.
   * @param transactional the Transactional for executing transactions
   * @param datasetName the name of the dataset to fix
   * @param doDisable whether to disable and re-enable explore first
   * @param partitionsPerTx how many partitions to process per transaction
   * @param verbose whether to log verbosely. If true, this will log a message for every partition; otherwise it
   *                will only log a report of how many partitions were added / could not be added.
   */
@Beta
@SuppressWarnings("unused")
public static void fixPartitions(Transactional transactional, final String datasetName, boolean doDisable, final int partitionsPerTx, final boolean verbose) {
    if (doDisable) {
        try {
            transactional.execute(new TxRunnable() {

                @Override
                public void run(co.cask.cdap.api.data.DatasetContext context) throws Exception {
                    PartitionedFileSetDataset pfs = context.getDataset(datasetName);
                    pfs.disableExplore();
                    // truncating = true, because this is like truncating
                    pfs.enableExplore(true);
                }
            });
        } catch (TransactionFailureException e) {
            throw new DataSetException("Unable to disable and enable Explore", e.getCause());
        } catch (RuntimeException e) {
            if (e.getCause() instanceof TransactionFailureException) {
                throw new DataSetException("Unable to disable and enable Explore", e.getCause().getCause());
            }
            throw e;
        }
    }
    final AtomicReference<PartitionKey> startKey = new AtomicReference<>();
    final AtomicLong errorCount = new AtomicLong(0L);
    final AtomicLong successCount = new AtomicLong(0L);
    do {
        try {
            transactional.execute(new TxRunnable() {

                @Override
                public void run(co.cask.cdap.api.data.DatasetContext context) throws Exception {
                    final PartitionedFileSetDataset pfs = context.getDataset(datasetName);
                    // compute start row for the scan, reset remembered start key to null
                    byte[] startRow = startKey.get() == null ? null : generateRowKey(startKey.get(), pfs.getPartitioning());
                    startKey.set(null);
                    PartitionConsumer consumer = new PartitionConsumer() {

                        int count = 0;

                        @Override
                        public void consume(PartitionKey key, String path, @Nullable PartitionMetadata metadata) {
                            if (count >= partitionsPerTx) {
                                // reached the limit: remember this key as the start for the next round
                                startKey.set(key);
                                return;
                            }
                            try {
                                pfs.addPartitionToExplore(key, path);
                                successCount.incrementAndGet();
                                if (verbose) {
                                    LOG.info("Added partition {} with path {}", key, path);
                                }
                            } catch (DataSetException e) {
                                errorCount.incrementAndGet();
                                if (verbose) {
                                    LOG.warn(e.getMessage(), e);
                                }
                            }
                            count++;
                        }
                    };
                    pfs.getPartitions(null, consumer, false, startRow, null, partitionsPerTx + 1);
                }
            });
        } catch (TransactionConflictException e) {
            throw new DataSetException("Transaction conflict while reading partitions. This should never happen. " + "Make sure that no other programs are using this dataset at the same time.");
        } catch (TransactionFailureException e) {
            throw new DataSetException("Transaction failure: " + e.getMessage(), e.getCause());
        } catch (RuntimeException e) {
            // this looks like duplication but is needed in case this is run from a worker: see CDAP-6837
            if (e.getCause() instanceof TransactionConflictException) {
                throw new DataSetException("Transaction conflict while reading partitions. This should never happen. " + "Make sure that no other programs are using this dataset at the same time.");
            } else if (e.getCause() instanceof TransactionFailureException) {
                throw new DataSetException("Transaction failure: " + e.getMessage(), e.getCause().getCause());
            } else {
                throw e;
            }
        }
    } while (// if it is null, then we consumed less than the limit in this round -> done
    startKey.get() != null);
    LOG.info("Added {} partitions, failed to add {} partitions.", successCount.get(), errorCount.get());
}