Examples with QueryHandle io.cdap.cdap.proto.QueryHandle used on opensource projects

Example 51 with QueryHandle

use of io.cdap.cdap.proto.QueryHandle in project cdap by caskdata.

the class HiveExploreServiceTimeoutTest method testTimeoutRunning.

@Test
public void testTimeoutRunning() throws Exception {
    Set<Long> beforeTxns = transactionManager.getCurrentState().getInProgress().keySet();
    QueryHandle handle = exploreService.execute(NAMESPACE_ID, "select key, value from " + MY_TABLE_NAME);
    Set<Long> queryTxns = Sets.difference(transactionManager.getCurrentState().getInProgress().keySet(), beforeTxns);
    Assert.assertFalse(queryTxns.isEmpty());
    // Sleep for timeout to happen
    TimeUnit.SECONDS.sleep(ACTIVE_OPERATION_TIMEOUT_SECS + 3);
    try {
        exploreService.getStatus(handle);
        Assert.fail("Should throw HandleNotFoundException due to operation timeout");
    } catch (HandleNotFoundException e) {
    // Expected exception due to timeout
    }
    // Make sure that the transaction got closed
    Assert.assertEquals(ImmutableSet.<Long>of(), Sets.intersection(queryTxns, transactionManager.getCurrentState().getInProgress().keySet()).immutableCopy());
}

Example 52 with QueryHandle

use of io.cdap.cdap.proto.QueryHandle in project cdap by caskdata.

the class HiveExploreServiceTimeoutTest method testCloseQuery.

@Test
public void testCloseQuery() throws Exception {
    QueryHandle handle = exploreService.execute(NAMESPACE_ID, "drop table if exists not_existing_table_name");
    exploreService.close(handle);
    try {
        exploreService.getStatus(handle);
        Assert.fail("Should throw HandleNotFoundException due to operation cleanup");
    } catch (HandleNotFoundException e) {
    // Expected exception due to timeout
    }
}

Example 53 with QueryHandle

use of io.cdap.cdap.proto.QueryHandle in project cdap by caskdata.

the class HiveExploreServiceTestRun method previewResultsTest.

@Test
public void previewResultsTest() throws Exception {
    DatasetId myTable2 = NAMESPACE_ID.dataset("my_table_2");
    DatasetId myTable3 = NAMESPACE_ID.dataset("my_table_3");
    DatasetId myTable4 = NAMESPACE_ID.dataset("my_table_4");
    DatasetId myTable5 = NAMESPACE_ID.dataset("my_table_5");
    DatasetId myTable6 = NAMESPACE_ID.dataset("my_table_6");
    datasetFramework.addInstance("keyStructValueTable", myTable2, DatasetProperties.EMPTY);
    datasetFramework.addInstance("keyStructValueTable", myTable3, DatasetProperties.EMPTY);
    datasetFramework.addInstance("keyStructValueTable", myTable4, DatasetProperties.EMPTY);
    datasetFramework.addInstance("keyStructValueTable", myTable5, DatasetProperties.EMPTY);
    datasetFramework.addInstance("keyStructValueTable", myTable6, DatasetProperties.EMPTY);
    try {
        QueryHandle handle = exploreService.execute(NAMESPACE_ID, "show tables");
        QueryStatus status = waitForCompletionStatus(handle, 200, TimeUnit.MILLISECONDS, 50);
        Assert.assertEquals(QueryStatus.OpStatus.FINISHED, status.getStatus());
        List<QueryResult> firstPreview = exploreService.previewResults(handle);
        Assert.assertEquals(ImmutableList.of(new QueryResult(ImmutableList.<Object>of(MY_TABLE_NAME)), new QueryResult(ImmutableList.<Object>of(getDatasetHiveName(myTable2))), new QueryResult(ImmutableList.<Object>of(getDatasetHiveName(myTable3))), new QueryResult(ImmutableList.<Object>of(getDatasetHiveName(myTable4))), new QueryResult(ImmutableList.<Object>of(getDatasetHiveName(myTable5)))), firstPreview);
        // test that preview results do not change even when the query cursor is updated by nextResults
        List<QueryResult> endResults = exploreService.nextResults(handle, 100);
        Assert.assertEquals(ImmutableList.of(new QueryResult(ImmutableList.<Object>of(getDatasetHiveName(myTable6)))), endResults);
        List<QueryResult> secondPreview = exploreService.previewResults(handle);
        Assert.assertEquals(firstPreview, secondPreview);
        Assert.assertEquals(ImmutableList.<QueryResult>of(), exploreService.nextResults(handle, 100));
        try {
            // All results are fetched, query should be inactive now
            exploreService.previewResults(handle);
            Assert.fail("HandleNotFoundException expected - query should be inactive.");
        } catch (HandleNotFoundException e) {
            Assert.assertTrue(e.isInactive());
        // Expected exception
        }
        // now test preview on a query that doesn't return any results
        handle = exploreService.execute(NAMESPACE_ID, "select * from " + getDatasetHiveName(myTable2));
        status = waitForCompletionStatus(handle, 200, TimeUnit.MILLISECONDS, 50);
        Assert.assertEquals(QueryStatus.OpStatus.FINISHED, status.getStatus());
        Assert.assertTrue(exploreService.previewResults(handle).isEmpty());
        // calling preview again should return the same thing. it should not throw an exception
        Assert.assertTrue(exploreService.previewResults(handle).isEmpty());
    } finally {
        datasetFramework.deleteInstance(myTable2);
        datasetFramework.deleteInstance(myTable3);
        datasetFramework.deleteInstance(myTable4);
        datasetFramework.deleteInstance(myTable5);
        datasetFramework.deleteInstance(myTable6);
    }
}

Example 54 with QueryHandle

use of io.cdap.cdap.proto.QueryHandle in project cdap by cdapio.

the class BaseHiveExploreService method doStartSession.

private Map<String, String> doStartSession(@Nullable NamespaceId namespace, @Nullable Map<String, String> additionalSessionConf) throws IOException, ExploreException, NamespaceNotFoundException {
    Map<String, String> sessionConf = new HashMap<>();
    QueryHandle queryHandle = QueryHandle.generate();
    sessionConf.put(Constants.Explore.QUERY_ID, queryHandle.getHandle());
    String schedulerQueue = namespace != null ? schedulerQueueResolver.getQueue(namespace) : schedulerQueueResolver.getDefaultQueue();
    if (schedulerQueue != null && !schedulerQueue.isEmpty()) {
        sessionConf.put(JobContext.QUEUE_NAME, schedulerQueue);
    }
    Transaction tx = startTransaction();
    ConfigurationUtil.set(sessionConf, Constants.Explore.TX_QUERY_KEY, TxnCodec.INSTANCE, tx);
    ConfigurationUtil.set(sessionConf, Constants.Explore.CCONF_KEY, CConfCodec.INSTANCE, cConf);
    ConfigurationUtil.set(sessionConf, Constants.Explore.HCONF_KEY, HConfCodec.INSTANCE, hConf);
    HiveConf hiveConf = createHiveConf();
    if (ExploreServiceUtils.isSparkEngine(hiveConf, additionalSessionConf)) {
        sessionConf.putAll(sparkConf);
    }
    if (UserGroupInformation.isSecurityEnabled()) {
        // make sure RM does not cancel delegation tokens after the query is run
        sessionConf.put("mapreduce.job.complete.cancel.delegation.tokens", "false");
        sessionConf.put("spark.hadoop.mapreduce.job.complete.cancel.delegation.tokens", "false");
        // write the user's credentials to a file, to be used for the query
        File credentialsFile = writeCredentialsFile(queryHandle);
        String credentialsFilePath = credentialsFile.getAbsolutePath();
        // mapreduce.job.credentials.binary is added by Hive only if Kerberos credentials are present and impersonation
        // is enabled. However, in our case we don't have Kerberos credentials for Explore service.
        // Hence it will not be automatically added by Hive, instead we have to add it ourselves.
        sessionConf.put(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, credentialsFilePath);
        // CDAP-8367 We need to set this back to Kerberos if security is enabled. We override it in HiveConf.
        sessionConf.put(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, UserGroupInformation.AuthenticationMethod.KERBEROS.name());
        sessionConf.put(Constants.Explore.SUBMITLOCALTASKVIACHILD, Boolean.FALSE.toString());
        sessionConf.put(Constants.Explore.SUBMITVIACHILD, Boolean.FALSE.toString());
        if (ExploreServiceUtils.isTezEngine(hiveConf, additionalSessionConf)) {
            // Add token file location property for tez if engine is tez
            sessionConf.put("tez.credentials.path", credentialsFilePath);
        }
    }
    if (additionalSessionConf != null) {
        sessionConf.putAll(additionalSessionConf);
    }
    return sessionConf;
}

Example 55 with QueryHandle

use of io.cdap.cdap.proto.QueryHandle in project cdap by cdapio.

the class BaseHiveExploreService method getQueries.

@Override
public List<QueryInfo> getQueries(NamespaceId namespace) throws ExploreException, SQLException {
    startAndWait();
    List<QueryInfo> result = new ArrayList<>();
    String namespaceHiveDb = getHiveDatabase(namespace.getNamespace());
    for (Map.Entry<QueryHandle, OperationInfo> entry : activeHandleCache.asMap().entrySet()) {
        try {
            if (namespaceHiveDb.equals(entry.getValue().getHiveDatabase())) {
                // we use empty query statement for get tables, get schemas, we don't need to return it this method call.
                if (!entry.getValue().getStatement().isEmpty()) {
                    QueryStatus status = getStatus(entry.getKey());
                    result.add(new QueryInfo(entry.getValue().getTimestamp(), entry.getValue().getStatement(), entry.getKey(), status, true));
                }
            }
        } catch (HandleNotFoundException e) {
        // ignore the handle not found exception. this method returns all queries and handle, if the
        // handle is removed from the internal cache, then there is no point returning them from here.
        }
    }
    for (Map.Entry<QueryHandle, InactiveOperationInfo> entry : inactiveHandleCache.asMap().entrySet()) {
        InactiveOperationInfo inactiveOperationInfo = entry.getValue();
        if (namespaceHiveDb.equals(inactiveOperationInfo.getHiveDatabase())) {
            // we use empty query statement for get tables, get schemas, we don't need to return it this method call.
            if (!inactiveOperationInfo.getStatement().isEmpty()) {
                if (inactiveOperationInfo.getStatus() == null) {
                    LOG.error("Null status for query {}, handle {}", inactiveOperationInfo.getStatement(), entry.getKey());
                }
                result.add(new QueryInfo(inactiveOperationInfo.getTimestamp(), inactiveOperationInfo.getStatement(), entry.getKey(), inactiveOperationInfo.getStatus(), false));
            }
        }
    }
    Collections.sort(result);
    return result;
}