Examples with Transaction org.apache.tephra.Transaction used on opensource projects

Example 31 with Transaction

use of org.apache.tephra.Transaction in project cdap by caskdata.

the class DequeueScanObserver method preScannerOpen.

@Override
public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s) throws IOException {
    ConsumerConfig consumerConfig = DequeueScanAttributes.getConsumerConfig(scan);
    Transaction tx = DequeueScanAttributes.getTx(scan);
    if (consumerConfig == null || tx == null) {
        return super.preScannerOpen(e, scan, s);
    }
    Filter dequeueFilter = new DequeueFilter(consumerConfig, tx);
    Filter existing = scan.getFilter();
    if (existing != null) {
        Filter combined = new FilterList(FilterList.Operator.MUST_PASS_ALL, existing, dequeueFilter);
        scan.setFilter(combined);
    } else {
        scan.setFilter(dequeueFilter);
    }
    return super.preScannerOpen(e, scan, s);
}

Example 32 with Transaction

use of org.apache.tephra.Transaction in project cdap by caskdata.

the class HiveExploreServiceTestRun method testJoin.

@Test
public void testJoin() throws Exception {
    DatasetId myTable1 = NAMESPACE_ID.dataset("my_table_1");
    String myTable1Name = getDatasetHiveName(myTable1);
    // Performing admin operations to create dataset instance
    datasetFramework.addInstance("keyStructValueTable", myTable1, DatasetProperties.EMPTY);
    try {
        Transaction tx1 = transactionManager.startShort(100);
        // Accessing dataset instance to perform data operations
        KeyStructValueTableDefinition.KeyStructValueTable table = datasetFramework.getDataset(myTable1, DatasetDefinition.NO_ARGUMENTS, null);
        Assert.assertNotNull(table);
        table.startTx(tx1);
        KeyValue.Value value1 = new KeyValue.Value("two", Lists.newArrayList(20, 21, 22, 23, 24));
        KeyValue.Value value2 = new KeyValue.Value("third", Lists.newArrayList(30, 31, 32, 33, 34));
        table.put("2", value1);
        table.put("3", value2);
        Assert.assertEquals(value1, table.get("2"));
        Assert.assertTrue(table.commitTx());
        transactionManager.canCommit(tx1.getTransactionId(), table.getTxChanges());
        transactionManager.commit(tx1.getTransactionId(), tx1.getWritePointer());
        table.postTxCommit();
        String query = String.format("select %s.key, %s.value from %s join %s on (%s.key=%s.key)", MY_TABLE_NAME, MY_TABLE_NAME, MY_TABLE_NAME, myTable1Name, MY_TABLE_NAME, myTable1Name);
        runCommand(NAMESPACE_ID, query, true, Lists.newArrayList(new ColumnDesc(MY_TABLE_NAME + ".key", "STRING", 1, null), new ColumnDesc(MY_TABLE_NAME + ".value", "struct<name:string,ints:array<int>>", 2, null)), Lists.newArrayList(new QueryResult(Lists.<Object>newArrayList("2", "{\"name\":\"two\",\"ints\":[10,11,12,13,14]}"))));
        query = String.format("select %s.key, %s.value, %s.key, %s.value " + "from %s right outer join %s on (%s.key=%s.key)", MY_TABLE_NAME, MY_TABLE_NAME, myTable1Name, myTable1Name, MY_TABLE_NAME, myTable1Name, MY_TABLE_NAME, myTable1Name);
        runCommand(NAMESPACE_ID, query, true, Lists.newArrayList(new ColumnDesc(MY_TABLE_NAME + ".key", "STRING", 1, null), new ColumnDesc(MY_TABLE_NAME + ".value", "struct<name:string,ints:array<int>>", 2, null), new ColumnDesc(myTable1Name + ".key", "STRING", 3, null), new ColumnDesc(myTable1Name + ".value", "struct<name:string,ints:array<int>>", 4, null)), Lists.newArrayList(new QueryResult(Lists.<Object>newArrayList("2", "{\"name\":\"two\",\"ints\":[10,11,12,13,14]}", "2", "{\"name\":\"two\",\"ints\":[20,21,22,23,24]}")), new QueryResult(Lists.<Object>newArrayList(null, null, "3", "{\"name\":\"third\",\"ints\":[30,31,32,33,34]}"))));
        query = String.format("select %s.key, %s.value, %s.key, %s.value from %s " + "left outer join %s on (%s.key=%s.key)", MY_TABLE_NAME, MY_TABLE_NAME, myTable1Name, myTable1Name, MY_TABLE_NAME, myTable1Name, MY_TABLE_NAME, myTable1Name);
        runCommand(NAMESPACE_ID, query, true, Lists.newArrayList(new ColumnDesc(MY_TABLE_NAME + ".key", "STRING", 1, null), new ColumnDesc(MY_TABLE_NAME + ".value", "struct<name:string,ints:array<int>>", 2, null), new ColumnDesc(myTable1Name + ".key", "STRING", 3, null), new ColumnDesc(myTable1Name + ".value", "struct<name:string,ints:array<int>>", 4, null)), Lists.newArrayList(new QueryResult(Lists.<Object>newArrayList("1", "{\"name\":\"first\",\"ints\":[1,2,3,4,5]}", null, null)), new QueryResult(Lists.<Object>newArrayList("2", "{\"name\":\"two\",\"ints\":[10,11,12,13,14]}", "2", "{\"name\":\"two\",\"ints\":[20,21,22,23,24]}"))));
        query = String.format("select %s.key, %s.value, %s.key, %s.value from %s " + "full outer join %s on (%s.key=%s.key)", MY_TABLE_NAME, MY_TABLE_NAME, myTable1Name, myTable1Name, MY_TABLE_NAME, myTable1Name, MY_TABLE_NAME, myTable1Name);
        runCommand(NAMESPACE_ID, query, true, Lists.newArrayList(new ColumnDesc(MY_TABLE_NAME + ".key", "STRING", 1, null), new ColumnDesc(MY_TABLE_NAME + ".value", "struct<name:string,ints:array<int>>", 2, null), new ColumnDesc(myTable1Name + ".key", "STRING", 3, null), new ColumnDesc(myTable1Name + ".value", "struct<name:string,ints:array<int>>", 4, null)), Lists.newArrayList(new QueryResult(Lists.<Object>newArrayList("1", "{\"name\":\"first\",\"ints\":[1,2,3,4,5]}", null, null)), new QueryResult(Lists.<Object>newArrayList("2", "{\"name\":\"two\",\"ints\":[10,11,12,13,14]}", "2", "{\"name\":\"two\",\"ints\":[20,21,22,23,24]}")), new QueryResult(Lists.<Object>newArrayList(null, null, "3", "{\"name\":\"third\",\"ints\":[30,31,32,33,34]}"))));
    } finally {
        datasetFramework.deleteInstance(myTable1);
    }
}

Example 33 with Transaction

use of org.apache.tephra.Transaction in project cdap by caskdata.

the class HiveExploreServiceTimeoutTest method start.

@BeforeClass
public static void start() throws Exception {
    // Set smaller values for timeouts for testing
    CConfiguration cConfiguration = CConfiguration.create();
    cConfiguration.setLong(Constants.Explore.ACTIVE_OPERATION_TIMEOUT_SECS, ACTIVE_OPERATION_TIMEOUT_SECS);
    cConfiguration.setLong(Constants.Explore.INACTIVE_OPERATION_TIMEOUT_SECS, INACTIVE_OPERATION_TIMEOUT_SECS);
    cConfiguration.setLong(Constants.Explore.CLEANUP_JOB_SCHEDULE_SECS, CLEANUP_JOB_SCHEDULE_SECS);
    initialize(cConfiguration, tmpFolder);
    exploreService = injector.getInstance(ExploreService.class);
    datasetFramework.addModule(KEY_STRUCT_VALUE, new KeyStructValueTableDefinition.KeyStructValueTableModule());
    // Performing admin operations to create dataset instance
    datasetFramework.addInstance("keyStructValueTable", MY_TABLE, DatasetProperties.EMPTY);
    // Accessing dataset instance to perform data operations
    KeyStructValueTableDefinition.KeyStructValueTable table = datasetFramework.getDataset(MY_TABLE, DatasetDefinition.NO_ARGUMENTS, null);
    Assert.assertNotNull(table);
    Transaction tx1 = transactionManager.startShort(100);
    table.startTx(tx1);
    KeyStructValueTableDefinition.KeyValue.Value value1 = new KeyStructValueTableDefinition.KeyValue.Value("first", Lists.newArrayList(1, 2, 3, 4, 5));
    KeyStructValueTableDefinition.KeyValue.Value value2 = new KeyStructValueTableDefinition.KeyValue.Value("two", Lists.newArrayList(10, 11, 12, 13, 14));
    table.put("1", value1);
    table.put("2", value2);
    Assert.assertEquals(value1, table.get("1"));
    Assert.assertTrue(table.commitTx());
    transactionManager.canCommit(tx1.getTransactionId(), table.getTxChanges());
    transactionManager.commit(tx1.getTransactionId(), tx1.getWritePointer());
    table.postTxCommit();
    Transaction tx2 = transactionManager.startShort(100);
    table.startTx(tx2);
    Assert.assertEquals(value1, table.get("1"));
}

Example 34 with Transaction

use of org.apache.tephra.Transaction in project cdap by caskdata.

the class BaseHiveExploreService method doStartSession.

private Map<String, String> doStartSession(@Nullable NamespaceId namespace, @Nullable Map<String, String> additionalSessionConf) throws IOException, ExploreException, NamespaceNotFoundException {
    Map<String, String> sessionConf = new HashMap<>();
    QueryHandle queryHandle = QueryHandle.generate();
    sessionConf.put(Constants.Explore.QUERY_ID, queryHandle.getHandle());
    String schedulerQueue = namespace != null ? schedulerQueueResolver.getQueue(Id.Namespace.fromEntityId(namespace)) : schedulerQueueResolver.getDefaultQueue();
    if (schedulerQueue != null && !schedulerQueue.isEmpty()) {
        sessionConf.put(JobContext.QUEUE_NAME, schedulerQueue);
    }
    Transaction tx = startTransaction();
    ConfigurationUtil.set(sessionConf, Constants.Explore.TX_QUERY_KEY, TxnCodec.INSTANCE, tx);
    ConfigurationUtil.set(sessionConf, Constants.Explore.CCONF_KEY, CConfCodec.INSTANCE, cConf);
    ConfigurationUtil.set(sessionConf, Constants.Explore.HCONF_KEY, HConfCodec.INSTANCE, hConf);
    HiveConf hiveConf = createHiveConf();
    if (ExploreServiceUtils.isSparkEngine(hiveConf, additionalSessionConf)) {
        sessionConf.putAll(sparkConf);
    }
    if (UserGroupInformation.isSecurityEnabled()) {
        // make sure RM does not cancel delegation tokens after the query is run
        sessionConf.put("mapreduce.job.complete.cancel.delegation.tokens", "false");
        sessionConf.put("spark.hadoop.mapreduce.job.complete.cancel.delegation.tokens", "false");
        // write the user's credentials to a file, to be used for the query
        File credentialsFile = writeCredentialsFile(queryHandle);
        String credentialsFilePath = credentialsFile.getAbsolutePath();
        // mapreduce.job.credentials.binary is added by Hive only if Kerberos credentials are present and impersonation
        // is enabled. However, in our case we don't have Kerberos credentials for Explore service.
        // Hence it will not be automatically added by Hive, instead we have to add it ourselves.
        sessionConf.put(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, credentialsFilePath);
        // CDAP-8367 We need to set this back to Kerberos if security is enabled. We override it in HiveConf.
        sessionConf.put(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, UserGroupInformation.AuthenticationMethod.KERBEROS.name());
        sessionConf.put(Constants.Explore.SUBMITLOCALTASKVIACHILD, Boolean.FALSE.toString());
        sessionConf.put(Constants.Explore.SUBMITVIACHILD, Boolean.FALSE.toString());
        if (ExploreServiceUtils.isTezEngine(hiveConf, additionalSessionConf)) {
            // Add token file location property for tez if engine is tez
            sessionConf.put("tez.credentials.path", credentialsFilePath);
        }
    }
    if (additionalSessionConf != null) {
        sessionConf.putAll(additionalSessionConf);
    }
    return sessionConf;
}

Example 35 with Transaction

use of org.apache.tephra.Transaction in project cdap by caskdata.

the class BaseHiveExploreService method startTransaction.

private Transaction startTransaction() throws IOException {
    Transaction tx = txClient.startLong();
    LOG.trace("Transaction {} started.", tx);
    return tx;
}