Examples with Transaction org.apache.tephra.Transaction used on opensource projects

Example 56 with Transaction

use of org.apache.tephra.Transaction in project cdap by caskdata.

the class SparkTransactionHandler method getTransaction.

/**
 * Handler method to get a serialized {@link Transaction} for the given stage.
 */
@GET
@Path("/spark/stages/{stage}/transaction")
public void getTransaction(HttpRequest request, HttpResponder responder, @PathParam("stage") int stageId) {
    // Lookup the jobId from the stageId
    Integer jobId = stageToJob.get(stageId);
    if (jobId == null) {
        // If the JobId is not there, it's either the job hasn't been registered yet (because it's async) or
        // the job is already finished. For either case, return 404 and let the client to handle retry if necessary.
        responder.sendString(HttpResponseStatus.NOT_FOUND, "JobId not found for stage " + stageId);
        return;
    }
    // Get the transaction
    JobTransaction jobTransaction = jobTransactions.get(jobId);
    if (jobTransaction == null) {
        // The only reason we can find the jobId from the stageToJob map but not the job transaction is because
        // the job is completed, hence the transaction get removed. In normal case, it shouldn't happen
        // as a job won't complete if there are still stages running and
        // this method only gets called from stage running in executor node.
        responder.sendString(HttpResponseStatus.GONE, "No transaction associated with the stage " + stageId + " of job " + jobId);
        return;
    }
    Transaction transaction = jobTransaction.getTransaction();
    if (transaction == null) {
        // Job failed to start a transaction. Response with GONE as well so that the stage execution can fail itself
        responder.sendString(HttpResponseStatus.GONE, "Failed to start transaction for stage " + stageId + " of job " + jobId);
        return;
    }
    // Serialize the transaction and send it back
    try {
        responder.sendByteArray(HttpResponseStatus.OK, TX_CODEC.encode(transaction), EmptyHttpHeaders.INSTANCE);
    } catch (IOException e) {
        // Shouldn't happen
        LOG.error("Failed to encode Transaction {}", jobTransaction, e);
        responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Failed to encode transaction: " + e.getMessage());
    }
}

Example 57 with Transaction

use of org.apache.tephra.Transaction in project cdap by caskdata.

the class HiveExploreTableTestRun method testInsertFromJoin.

@Test
public void testInsertFromJoin() throws Exception {
    DatasetId userTableID = NAMESPACE_ID.dataset("users");
    DatasetId purchaseTableID = NAMESPACE_ID.dataset("purchases");
    DatasetId expandedTableID = NAMESPACE_ID.dataset("expanded");
    Schema userSchema = Schema.recordOf("user", Schema.Field.of("id", Schema.of(Schema.Type.STRING)), Schema.Field.of("name", Schema.of(Schema.Type.STRING)), Schema.Field.of("email", Schema.of(Schema.Type.STRING)));
    Schema purchaseSchema = Schema.recordOf("purchase", Schema.Field.of("purchaseid", Schema.of(Schema.Type.LONG)), Schema.Field.of("itemid", Schema.of(Schema.Type.STRING)), Schema.Field.of("userid", Schema.of(Schema.Type.STRING)), Schema.Field.of("ct", Schema.of(Schema.Type.INT)), Schema.Field.of("price", Schema.of(Schema.Type.DOUBLE)));
    Schema expandedSchema = Schema.recordOf("expandedPurchase", Schema.Field.of("purchaseid", Schema.of(Schema.Type.LONG)), Schema.Field.of("itemid", Schema.of(Schema.Type.STRING)), Schema.Field.of("userid", Schema.of(Schema.Type.STRING)), Schema.Field.of("ct", Schema.of(Schema.Type.INT)), Schema.Field.of("price", Schema.of(Schema.Type.DOUBLE)), Schema.Field.of("username", Schema.of(Schema.Type.STRING)), Schema.Field.of("email", Schema.of(Schema.Type.STRING)));
    datasetFramework.addInstance(Table.class.getName(), userTableID, TableProperties.builder().setSchema(userSchema).setRowFieldName("id").build());
    datasetFramework.addInstance(Table.class.getName(), purchaseTableID, TableProperties.builder().setSchema(purchaseSchema).setRowFieldName("purchaseid").build());
    datasetFramework.addInstance(Table.class.getName(), expandedTableID, TableProperties.builder().setSchema(expandedSchema).setRowFieldName("purchaseid").build());
    Table userTable = datasetFramework.getDataset(userTableID, DatasetDefinition.NO_ARGUMENTS, null);
    Table purchaseTable = datasetFramework.getDataset(purchaseTableID, DatasetDefinition.NO_ARGUMENTS, null);
    TransactionAware txUserTable = (TransactionAware) userTable;
    TransactionAware txPurchaseTable = (TransactionAware) purchaseTable;
    Transaction tx1 = transactionManager.startShort(100);
    txUserTable.startTx(tx1);
    txPurchaseTable.startTx(tx1);
    Put put = new Put(Bytes.toBytes("samuel"));
    put.add("name", "Samuel Jackson");
    put.add("email", "sjackson@gmail.com");
    userTable.put(put);
    put = new Put(Bytes.toBytes(1L));
    put.add("userid", "samuel");
    put.add("itemid", "scotch");
    put.add("ct", 1);
    put.add("price", 56.99d);
    purchaseTable.put(put);
    txUserTable.commitTx();
    txPurchaseTable.commitTx();
    List<byte[]> changes = new ArrayList<>();
    changes.addAll(txUserTable.getTxChanges());
    changes.addAll(txPurchaseTable.getTxChanges());
    transactionManager.canCommit(tx1.getTransactionId(), changes);
    transactionManager.commit(tx1.getTransactionId(), tx1.getWritePointer());
    txUserTable.postTxCommit();
    txPurchaseTable.postTxCommit();
    try {
        String command = String.format("insert into table %s select P.purchaseid, P.itemid, P.userid, P.ct, P.price, U.name, U.email from " + "%s P join %s U on (P.userid = U.id)", getDatasetHiveName(expandedTableID), getDatasetHiveName(purchaseTableID), getDatasetHiveName(userTableID));
        ExploreExecutionResult result = exploreClient.submit(NAMESPACE_ID, command).get();
        Assert.assertEquals(QueryStatus.OpStatus.FINISHED, result.getStatus().getStatus());
        command = String.format("select purchaseid, itemid, userid, ct, price, username, email from %s", getDatasetHiveName(expandedTableID));
        runCommand(NAMESPACE_ID, command, true, Lists.newArrayList(new ColumnDesc("purchaseid", "BIGINT", 1, null), new ColumnDesc("itemid", "STRING", 2, null), new ColumnDesc("userid", "STRING", 3, null), new ColumnDesc("ct", "INT", 4, null), new ColumnDesc("price", "DOUBLE", 5, null), new ColumnDesc("username", "STRING", 6, null), new ColumnDesc("email", "STRING", 7, null)), Lists.newArrayList(new QueryResult(Lists.<Object>newArrayList(1L, "scotch", "samuel", 1, 56.99d, "Samuel Jackson", "sjackson@gmail.com"))));
    } finally {
        datasetFramework.deleteInstance(userTableID);
        datasetFramework.deleteInstance(purchaseTableID);
        datasetFramework.deleteInstance(expandedTableID);
    }
}

Example 58 with Transaction

use of org.apache.tephra.Transaction in project cdap by caskdata.

the class HiveExploreTableTestRun method setupTable.

private void setupTable(@Nullable String dbName, @Nullable String tableName) throws Exception {
    if (dbName != null) {
        runCommand(NAMESPACE_ID, "create database if not exists " + dbName, false, null, null);
    }
    datasetFramework.addInstance(Table.class.getName(), MY_TABLE, setupTableProperties(dbName, tableName, SCHEMA));
    // Accessing dataset instance to perform data operations
    Table table = datasetFramework.getDataset(MY_TABLE, DatasetDefinition.NO_ARGUMENTS, null);
    Assert.assertNotNull(table);
    TransactionAware txTable = (TransactionAware) table;
    Transaction tx1 = transactionManager.startShort(100);
    txTable.startTx(tx1);
    Put put = new Put(Bytes.toBytes("row1"));
    put.add("bool_field", false);
    put.add("int_field", Integer.MAX_VALUE);
    put.add("long_field", Long.MAX_VALUE);
    put.add("float_field", 3.14f);
    put.add("double_field", 3.14);
    put.add("bytes_field", new byte[] { 'A', 'B', 'C' });
    table.put(put);
    Assert.assertTrue(txTable.commitTx());
    transactionManager.canCommit(tx1.getTransactionId(), txTable.getTxChanges());
    transactionManager.commit(tx1.getTransactionId(), tx1.getWritePointer());
    txTable.postTxCommit();
    Transaction tx2 = transactionManager.startShort(100);
    txTable.startTx(tx2);
}

Example 59 with Transaction

use of org.apache.tephra.Transaction in project cdap by caskdata.

the class WritableDatasetTestRun method writeFromAnotherNamespace.

@Test
public void writeFromAnotherNamespace() throws Exception {
    datasetFramework.addModule(kvTable, new KeyValueTableDefinition.KeyValueTableModule());
    datasetFramework.addInstance("kvTable", simpleTable, DatasetProperties.EMPTY);
    datasetFramework.addModule(otherKvTable, new KeyValueTableDefinition.KeyValueTableModule());
    datasetFramework.addInstance("kvTable", otherSimpleTable, DatasetProperties.EMPTY);
    try {
        ExploreExecutionResult result = exploreClient.submit(OTHER_NAMESPACE_ID, "select * from " + simpleTableName).get();
        Assert.assertFalse(result.hasNext());
        // Accessing dataset instance to perform data operations
        KeyValueTableDefinition.KeyValueTable table = datasetFramework.getDataset(simpleTable, DatasetDefinition.NO_ARGUMENTS, null);
        Assert.assertNotNull(table);
        Transaction tx = transactionManager.startShort(100);
        table.startTx(tx);
        table.put(1, "one");
        Assert.assertTrue(table.commitTx());
        transactionManager.canCommit(tx.getTransactionId(), table.getTxChanges());
        transactionManager.commit(tx.getTransactionId(), tx.getWritePointer());
        table.postTxCommit();
        String query = String.format("insert into table %s select * from cdap_namespace.%s", otherSimpleTableName, simpleTableName);
        exploreClient.submit(OTHER_NAMESPACE_ID, query).get().close();
        assertSelectAll(NAMESPACE_ID, simpleTableName, ImmutableList.<List<Object>>of(ImmutableList.<Object>of(1, "one")));
        // Write into otherSimpleTable and assert that it doesn't show up in queries over simpleTable
        table = datasetFramework.getDataset(otherSimpleTable, DatasetDefinition.NO_ARGUMENTS, null);
        Assert.assertNotNull(table);
        tx = transactionManager.startShort(100);
        table.startTx(tx);
        table.put(2, "two");
        Assert.assertTrue(table.commitTx());
        transactionManager.canCommit(tx.getTransactionId(), table.getTxChanges());
        transactionManager.commit(tx.getTransactionId(), tx.getWritePointer());
        table.postTxCommit();
        assertSelectAll(OTHER_NAMESPACE_ID, otherSimpleTableName, ImmutableList.<List<Object>>of(ImmutableList.<Object>of(1, "one"), ImmutableList.<Object>of(2, "two")));
        assertSelectAll(NAMESPACE_ID, simpleTableName, ImmutableList.<List<Object>>of(ImmutableList.<Object>of(1, "one")));
    } finally {
        datasetFramework.deleteInstance(simpleTable);
        datasetFramework.deleteInstance(otherSimpleTable);
        datasetFramework.deleteModule(kvTable);
        datasetFramework.deleteModule(otherKvTable);
    }
}

Example 60 with Transaction

use of org.apache.tephra.Transaction in project cdap by caskdata.

the class WritableDatasetTestRun method writeIntoNonScannableDataset.

@Test
public void writeIntoNonScannableDataset() throws Exception {
    DatasetId writableTable = NAMESPACE_ID.dataset("writable_table");
    String writableTableName = getDatasetHiveName(writableTable);
    datasetFramework.addModule(keyExtendedStructValueTable, new KeyExtendedStructValueTableDefinition.KeyExtendedStructValueTableModule());
    datasetFramework.addInstance("keyExtendedStructValueTable", extendedTable, DatasetProperties.EMPTY);
    datasetFramework.addModule(writableKeyStructValueTable, new WritableKeyStructValueTableDefinition.KeyStructValueTableModule());
    datasetFramework.addInstance("writableKeyStructValueTable", writableTable, DatasetProperties.EMPTY);
    try {
        // Accessing dataset instance to perform data operations
        KeyExtendedStructValueTableDefinition.KeyExtendedStructValueTable table = datasetFramework.getDataset(extendedTable, DatasetDefinition.NO_ARGUMENTS, null);
        Assert.assertNotNull(table);
        Transaction tx1 = transactionManager.startShort(100);
        table.startTx(tx1);
        KeyExtendedStructValueTableDefinition.KeyExtendedValue value1 = new KeyExtendedStructValueTableDefinition.KeyExtendedValue("10", new KeyStructValueTableDefinition.KeyValue.Value("ten", Lists.newArrayList(10, 11, 12)), 20);
        table.put("10", value1);
        Assert.assertEquals(value1, table.get("10"));
        Assert.assertTrue(table.commitTx());
        transactionManager.canCommit(tx1.getTransactionId(), table.getTxChanges());
        transactionManager.commit(tx1.getTransactionId(), tx1.getWritePointer());
        table.postTxCommit();
        String query = "insert into table " + writableTableName + " select key,value from " + extendedTableName;
        ListenableFuture<ExploreExecutionResult> future = exploreClient.submit(NAMESPACE_ID, query);
        ExploreExecutionResult result = future.get();
        result.close();
        KeyStructValueTableDefinition.KeyStructValueTable table2 = datasetFramework.getDataset(writableTable, DatasetDefinition.NO_ARGUMENTS, null);
        Assert.assertNotNull(table);
        Transaction tx = transactionManager.startShort(100);
        Assert.assertNotNull(table2);
        table2.startTx(tx);
        Assert.assertEquals(new KeyStructValueTableDefinition.KeyValue.Value("ten", Lists.newArrayList(10, 11, 12)), table2.get("10_2"));
        Assert.assertTrue(table.commitTx());
        transactionManager.canCommit(tx.getTransactionId(), table.getTxChanges());
        transactionManager.commit(tx.getTransactionId(), tx.getWritePointer());
        table.postTxCommit();
    } finally {
        datasetFramework.deleteInstance(writableTable);
        datasetFramework.deleteInstance(extendedTable);
        datasetFramework.deleteModule(writableKeyStructValueTable);
        datasetFramework.deleteModule(keyExtendedStructValueTable);
    }
}