Examples with Transaction org.apache.tephra.Transaction used on opensource projects

Example 1 with Transaction

use of org.apache.tephra.Transaction in project phoenix by apache.

the class PhoenixTransactionalIndexer method getIndexUpdates.

private Collection<Pair<Mutation, byte[]>> getIndexUpdates(RegionCoprocessorEnvironment env, PhoenixIndexMetaData indexMetaData, Iterator<Mutation> mutationIterator, byte[] txRollbackAttribute) throws IOException {
    Transaction tx = indexMetaData.getTransaction();
    if (tx == null) {
        throw new NullPointerException("Expected to find transaction in metadata for " + env.getRegionInfo().getTable().getNameAsString());
    }
    boolean isRollback = txRollbackAttribute != null;
    boolean isImmutable = indexMetaData.isImmutableRows();
    ResultScanner currentScanner = null;
    TransactionAwareHTable txTable = null;
    // Collect up all mutations in batch
    Map<ImmutableBytesPtr, MultiMutation> mutations = new HashMap<ImmutableBytesPtr, MultiMutation>();
    Map<ImmutableBytesPtr, MultiMutation> findPriorValueMutations;
    if (isImmutable && !isRollback) {
        findPriorValueMutations = new HashMap<ImmutableBytesPtr, MultiMutation>();
    } else {
        findPriorValueMutations = mutations;
    }
    while (mutationIterator.hasNext()) {
        Mutation m = mutationIterator.next();
        // add the mutation to the batch set
        ImmutableBytesPtr row = new ImmutableBytesPtr(m.getRow());
        if (mutations != findPriorValueMutations && isDeleteMutation(m)) {
            addMutation(findPriorValueMutations, row, m);
        }
        addMutation(mutations, row, m);
    }
    // Collect the set of mutable ColumnReferences so that we can first
    // run a scan to get the current state. We'll need this to delete
    // the existing index rows.
    List<IndexMaintainer> indexMaintainers = indexMetaData.getIndexMaintainers();
    int estimatedSize = indexMaintainers.size() * 10;
    Set<ColumnReference> mutableColumns = Sets.newHashSetWithExpectedSize(estimatedSize);
    for (IndexMaintainer indexMaintainer : indexMaintainers) {
        // For transactional tables, we use an index maintainer
        // to aid in rollback if there's a KeyValue column in the index. The alternative would be
        // to hold on to all uncommitted index row keys (even ones already sent to HBase) on the
        // client side.
        Set<ColumnReference> allColumns = indexMaintainer.getAllColumns();
        mutableColumns.addAll(allColumns);
    }
    Collection<Pair<Mutation, byte[]>> indexUpdates = new ArrayList<Pair<Mutation, byte[]>>(mutations.size() * 2 * indexMaintainers.size());
    try {
        // this logic will work there too.
        if (!findPriorValueMutations.isEmpty()) {
            List<KeyRange> keys = Lists.newArrayListWithExpectedSize(mutations.size());
            for (ImmutableBytesPtr ptr : findPriorValueMutations.keySet()) {
                keys.add(PVarbinary.INSTANCE.getKeyRange(ptr.copyBytesIfNecessary()));
            }
            Scan scan = new Scan();
            // Project all mutable columns
            for (ColumnReference ref : mutableColumns) {
                scan.addColumn(ref.getFamily(), ref.getQualifier());
            }
            /*
                 * Indexes inherit the storage scheme of the data table which means all the indexes have the same
                 * storage scheme and empty key value qualifier. Note that this assumption would be broken if we start
                 * supporting new indexes over existing data tables to have a different storage scheme than the data
                 * table.
                 */
            byte[] emptyKeyValueQualifier = indexMaintainers.get(0).getEmptyKeyValueQualifier();
            // Project empty key value column
            scan.addColumn(indexMaintainers.get(0).getDataEmptyKeyValueCF(), emptyKeyValueQualifier);
            ScanRanges scanRanges = ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, KeyRange.EVERYTHING_RANGE, null, true, -1);
            scanRanges.initializeScan(scan);
            TableName tableName = env.getRegion().getRegionInfo().getTable();
            HTableInterface htable = env.getTable(tableName);
            txTable = new TransactionAwareHTable(htable);
            txTable.startTx(tx);
            // For rollback, we need to see all versions, including
            // the last committed version as there may be multiple
            // checkpointed versions.
            SkipScanFilter filter = scanRanges.getSkipScanFilter();
            if (isRollback) {
                filter = new SkipScanFilter(filter, true);
                tx.setVisibility(VisibilityLevel.SNAPSHOT_ALL);
            }
            scan.setFilter(filter);
            currentScanner = txTable.getScanner(scan);
        }
        if (isRollback) {
            processRollback(env, indexMetaData, txRollbackAttribute, currentScanner, tx, mutableColumns, indexUpdates, mutations);
        } else {
            processMutation(env, indexMetaData, txRollbackAttribute, currentScanner, tx, mutableColumns, indexUpdates, mutations, findPriorValueMutations);
        }
    } finally {
        if (txTable != null)
            txTable.close();
    }
    return indexUpdates;
}

Example 2 with Transaction

use of org.apache.tephra.Transaction in project cdap by caskdata.

the class CDAPOperationalStatsTest method setup.

@BeforeClass
public static void setup() throws Exception {
    injector = AppFabricTestHelper.getInjector();
    namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(NAMESPACE).build());
    CConfiguration cConf = injector.getInstance(CConfiguration.class);
    AppFabricTestHelper.deployApplication(Id.Namespace.fromEntityId(NAMESPACE), AllProgramsApp.class, null, cConf);
    TransactionSystemClient txClient = injector.getInstance(TransactionSystemClient.class);
    Transaction tx1 = txClient.startShort();
    txClient.canCommitOrThrow(tx1, Collections.singleton(Bytes.toBytes("foo")));
    Transaction tx2 = txClient.startShort();
    txClient.commitOrThrow(tx2);
    Transaction tx3 = txClient.startShort();
    txClient.invalidate(tx3.getTransactionId());
}

Example 3 with Transaction

use of org.apache.tephra.Transaction in project cdap by caskdata.

the class SparkTransactionHandlerTest method testExplicitTransaction.

/**
 * Tests the explicit transaction that covers multiple jobs.
 */
@Test
public void testExplicitTransaction() throws Exception {
    final AtomicInteger jobIdGen = new AtomicInteger();
    final AtomicInteger stageIdGen = new AtomicInteger();
    Transaction transaction = txClient.startLong();
    // Execute two jobs with the same explicit transaction
    testRunJob(jobIdGen.getAndIncrement(), generateStages(stageIdGen, 2), true, transaction);
    testRunJob(jobIdGen.getAndIncrement(), generateStages(stageIdGen, 3), true, transaction);
    // Should be able to commit the transaction
    txClient.commitOrThrow(transaction);
}

Example 4 with Transaction

use of org.apache.tephra.Transaction in project cdap by caskdata.

the class SparkTransactionHandlerTest method testRunJob.

/**
 * Simulates a single job run which contains multiple stages with an optional explicit {@link Transaction} to use.
 *
 * @param jobId the job id
 * @param stages stages of the job
 * @param jobSucceeded end result of the job
 * @param explicitTransaction the job transaction to use if not {@code null}
 */
private void testRunJob(int jobId, Set<Integer> stages, boolean jobSucceeded, @Nullable final Transaction explicitTransaction) throws Exception {
    // Before job start, no transaction will be associated with the stages
    verifyStagesTransactions(stages, new ClientTransactionVerifier() {

        @Override
        public boolean verify(@Nullable Transaction transaction, @Nullable Throwable failureCause) throws Exception {
            return transaction == null && failureCause instanceof TimeoutException;
        }
    });
    // Now start the job
    if (explicitTransaction == null) {
        sparkTxHandler.jobStarted(jobId, stages);
    } else {
        sparkTxHandler.jobStarted(jobId, stages, new TransactionInfo() {

            @Override
            public Transaction getTransaction() {
                return explicitTransaction;
            }

            @Override
            public boolean commitOnJobEnded() {
                return false;
            }

            @Override
            public void onJobStarted() {
            // no-op
            }

            @Override
            public void onTransactionCompleted(boolean jobSucceeded, @Nullable TransactionFailureException failureCause) {
            // no-op
            }
        });
    }
    // For all stages, it should get the same transaction
    final Set<Transaction> transactions = Collections.newSetFromMap(new ConcurrentHashMap<Transaction, Boolean>());
    verifyStagesTransactions(stages, new ClientTransactionVerifier() {

        @Override
        public boolean verify(@Nullable Transaction transaction, @Nullable Throwable failureCause) throws Exception {
            transactions.add(new TransactionWrapper(transaction));
            return transaction != null;
        }
    });
    // Transactions returned for all stages belonging to the same job must return the same transaction
    Assert.assertEquals(1, transactions.size());
    // The transaction must be in progress
    Transaction transaction = transactions.iterator().next();
    Assert.assertTrue(txManager.getCurrentState().getInProgress().containsKey(transaction.getWritePointer()));
    // If run with an explicit transaction, then all stages' transactions must be the same as the explicit transaction
    if (explicitTransaction != null) {
        Assert.assertEquals(new TransactionWrapper(explicitTransaction), transaction);
    }
    // Now finish the job
    sparkTxHandler.jobEnded(jobId, jobSucceeded);
    // After job finished, no transaction will be associated with the stages
    verifyStagesTransactions(stages, new ClientTransactionVerifier() {

        @Override
        public boolean verify(@Nullable Transaction transaction, @Nullable Throwable failureCause) throws Exception {
            return transaction == null && failureCause instanceof TimeoutException;
        }
    });
    // Check the transaction state based on the job result
    TransactionSnapshot txState = txManager.getCurrentState();
    // If explicit transaction is used, the transaction should still be in-progress
    if (explicitTransaction != null) {
        Assert.assertTrue(txState.getInProgress().containsKey(transaction.getWritePointer()));
    } else {
        // With implicit transaction, after job completed, the tx shouldn't be in-progress
        Assert.assertFalse(txState.getInProgress().containsKey(transaction.getWritePointer()));
        if (jobSucceeded) {
            // Transaction must not be in the invalid list
            Assert.assertFalse(txState.getInvalid().contains(transaction.getWritePointer()));
        } else {
            // Transaction must be in the invalid list
            Assert.assertTrue(txState.getInvalid().contains(transaction.getWritePointer()));
        }
    }
}

Example 5 with Transaction

use of org.apache.tephra.Transaction in project cdap by caskdata.

the class SparkTransactionClient method getTransaction.

/**
 * Returns the {@link Transaction} for the given stage.
 *
 * @param stageId the stage id to query for {@link Transaction}.
 * @param timeout the maximum time to wait
 * @param timeUnit the time unit of the timeout argument
 * @return the {@link Transaction} to be used for the given stage.
 *
 * @throws TimeoutException if the wait timed out
 * @throws InterruptedException if the current thread was interrupted while waiting
 * @throws TransactionFailureException if failed to get transaction for the given stage. Calling this method again
 *                                     with the same stage id will result in the same exception
 */
public Transaction getTransaction(int stageId, long timeout, TimeUnit timeUnit) throws TimeoutException, InterruptedException, TransactionFailureException {
    long timeoutMillis = Math.max(0L, timeUnit.toMillis(timeout) - txPollIntervalMillis);
    Stopwatch stopwatch = new Stopwatch().start();
    Transaction transaction = getTransaction(stageId);
    while (transaction == null && stopwatch.elapsedMillis() < timeoutMillis) {
        TimeUnit.MILLISECONDS.sleep(txPollIntervalMillis);
        transaction = getTransaction(stageId);
    }
    if (transaction == null) {
        throw new TimeoutException("Cannot get transaction for stage " + stageId + " after " + timeout + " " + timeUnit);
    }
    return transaction;
}