Examples with ScheduledQuery org.apache.hadoop.hive.metastore.api.ScheduledQuery used on opensource projects

Example 16 with ScheduledQuery

use of org.apache.hadoop.hive.metastore.api.ScheduledQuery in project hive by apache.

the class ScheduledQueryAnalyzer method analyzeInternal.

@Override
public void analyzeInternal(ASTNode ast) throws SemanticException {
    ScheduledQueryMaintenanceWork work;
    ScheduledQueryMaintenanceRequestType type = translateAstType(ast.getToken().getType());
    ScheduledQuery parsedSchq = interpretAstNode(ast);
    ScheduledQuery schq = fillScheduledQuery(type, parsedSchq);
    checkAuthorization(type, schq);
    LOG.info("scheduled query operation: " + type + " " + schq);
    try {
        schq.validate();
    } catch (TException e) {
        throw new SemanticException("ScheduledQuery is invalid", e);
    }
    work = new ScheduledQueryMaintenanceWork(type, schq);
    rootTasks.add(TaskFactory.get(work));
    queryState.setCommandType(toHiveOperation(type));
}

Example 17 with ScheduledQuery

use of org.apache.hadoop.hive.metastore.api.ScheduledQuery in project hive by apache.

the class TestMetastoreScheduledQueries method testDuplicateCreate.

@Test(expected = AlreadyExistsException.class)
public void testDuplicateCreate() throws Exception {
    ScheduledQuery schq = createScheduledQuery(createKey("duplicate", "c1"));
    ScheduledQueryMaintenanceRequest r = new ScheduledQueryMaintenanceRequest();
    r.setType(ScheduledQueryMaintenanceRequestType.CREATE);
    r.setScheduledQuery(schq);
    client.scheduledQueryMaintenance(r);
    client.scheduledQueryMaintenance(r);
}

Example 18 with ScheduledQuery

use of org.apache.hadoop.hive.metastore.api.ScheduledQuery in project hive by apache.

the class TestMetastoreScheduledQueries method testPoll.

@Test
public void testPoll() throws Exception {
    ScheduledQuery schq = createScheduledQuery(new ScheduledQueryKey("q1", "polltest"));
    ScheduledQueryMaintenanceRequest r = new ScheduledQueryMaintenanceRequest();
    r.setType(ScheduledQueryMaintenanceRequestType.CREATE);
    r.setScheduledQuery(schq);
    client.scheduledQueryMaintenance(r);
    schq.setScheduleKey(new ScheduledQueryKey("q1", "polltestOther"));
    client.scheduledQueryMaintenance(r);
    // disabled queries are not considered
    schq.setScheduleKey(new ScheduledQueryKey("q2disabled", "polltest"));
    schq.setEnabled(false);
    client.scheduledQueryMaintenance(r);
    // do some poll requests; and wait for q1's execution
    ScheduledQueryPollRequest request = new ScheduledQueryPollRequest();
    request.setClusterNamespace("polltest");
    ScheduledQueryPollResponse pollResult = null;
    // wait for poll to hit
    for (int i = 0; i < 30; i++) {
        pollResult = client.scheduledQueryPoll(request);
        if (pollResult.isSetQuery()) {
            break;
        }
        Thread.sleep(100);
    }
    assertTrue(pollResult.isSetQuery());
    assertTrue(pollResult.isSetScheduleKey());
    assertTrue(pollResult.isSetExecutionId());
    // after reading the only scheduled query; there are no more queries to run (for 1 sec)
    ScheduledQueryPollResponse pollResult2 = client.scheduledQueryPoll(request);
    assertTrue(!pollResult2.isSetQuery());
    try (PersistenceManager pm = PersistenceManagerProvider.getPersistenceManager()) {
        MScheduledExecution q = pm.getObjectById(MScheduledExecution.class, pollResult.getExecutionId());
        assertNotNull(q);
        assertEquals(QueryState.INITED, q.getState());
        assertTrue(q.getStartTime() <= getEpochSeconds());
        assertTrue(q.getStartTime() >= getEpochSeconds() - 1);
        assertTrue(q.getEndTime() == null);
        assertTrue(q.getLastUpdateTime() <= getEpochSeconds());
        assertTrue(q.getLastUpdateTime() >= getEpochSeconds() - 1);
    }
    // wait 1 sec
    Thread.sleep(1000);
    ScheduledQueryProgressInfo info;
    info = new ScheduledQueryProgressInfo(pollResult.getExecutionId(), QueryState.EXECUTING, "executor-query-id");
    client.scheduledQueryProgress(info);
    try (PersistenceManager pm = PersistenceManagerProvider.getPersistenceManager()) {
        MScheduledExecution q = pm.getObjectById(MScheduledExecution.class, pollResult.getExecutionId());
        assertEquals(QueryState.EXECUTING, q.getState());
        assertEquals("executor-query-id", q.getExecutorQueryId());
        assertTrue(q.getLastUpdateTime() <= getEpochSeconds());
        assertTrue(q.getLastUpdateTime() >= getEpochSeconds() - 1);
    }
    // wait 1 sec
    Thread.sleep(1000);
    info = new ScheduledQueryProgressInfo(pollResult.getExecutionId(), QueryState.FAILED, "executor-query-id");
    info.setErrorMessage(generateLongErrorMessage());
    // info.set
    client.scheduledQueryProgress(info);
    try (PersistenceManager pm = PersistenceManagerProvider.getPersistenceManager()) {
        MScheduledExecution q = pm.getObjectById(MScheduledExecution.class, pollResult.getExecutionId());
        assertEquals(QueryState.FAILED, q.getState());
        assertEquals("executor-query-id", q.getExecutorQueryId());
        assertNull(q.getLastUpdateTime());
        assertTrue(q.getEndTime() <= getEpochSeconds());
        assertTrue(q.getEndTime() >= getEpochSeconds() - 1);
        assertTrue(q.getErrorMessage().length() < 2000);
        assertFalse(q.getErrorMessage().contains("x"));
    }
    // clustername is taken into account; this should be empty
    request.setClusterNamespace("polltestSomethingElse");
    pollResult = client.scheduledQueryPoll(request);
    assertFalse(pollResult.isSetQuery());
}

Example 19 with ScheduledQuery

use of org.apache.hadoop.hive.metastore.api.ScheduledQuery in project hive by apache.

the class TestMetastoreScheduledQueries method testDisableInternal.

/**
 * Simulates some schq failure scenario.
 *
 * Q1 is executed correctly 1 time; but every further executions are end with failures
 * Q2 should be executed without without issues even thru Q1 will be disabled at some point
 */
public void testDisableInternal(int q1NumberOfExecutions, int seconds, String testNamespace) throws Exception {
    int q2NumberOfExecutions = seconds - 1;
    ScheduledQueryKey schqKey1 = new ScheduledQueryKey("q1", testNamespace);
    ScheduledQueryKey schqKey2 = new ScheduledQueryKey("q2", testNamespace);
    createEverySecondSchq(schqKey1);
    createEverySecondSchq(schqKey2);
    // do some polls and report failure each time
    ScheduledQueryPollRequest request = new ScheduledQueryPollRequest();
    request.setClusterNamespace(testNamespace);
    ScheduledQueryPollResponse pollResult = null;
    int idx1 = 0;
    int idx2 = 0;
    for (int i = 0; i < seconds * 10 + 9; i++) {
        pollResult = client.scheduledQueryPoll(request);
        if (pollResult.isSetQuery()) {
            if (pollResult.getScheduleKey().equals(schqKey1)) {
                idx1++;
                if (idx1 == 1) {
                    ScheduledQueryProgressInfo info = new ScheduledQueryProgressInfo(pollResult.getExecutionId(), QueryState.FINISHED, "executor-query-id");
                    client.scheduledQueryProgress(info);
                } else {
                    ScheduledQueryProgressInfo info = new ScheduledQueryProgressInfo(pollResult.getExecutionId(), QueryState.FAILED, "executor-query-id");
                    info.setErrorMessage("some issue happened");
                    client.scheduledQueryProgress(info);
                }
                ScheduledQuery schq = client.getScheduledQuery(pollResult.getScheduleKey());
                if (idx1 > q1NumberOfExecutions) {
                    fail("unexpected execution of q1 happened");
                }
                if (idx1 == q1NumberOfExecutions) {
                    assertFalse("Scheduled query q1 must be disabled at this point", schq.isEnabled());
                } else {
                    assertTrue("Scheduled query q1 must be enabled at this point", schq.isEnabled());
                }
            }
            if (pollResult.getScheduleKey().equals(schqKey2)) {
                idx2++;
                ScheduledQueryProgressInfo info = new ScheduledQueryProgressInfo(pollResult.getExecutionId(), QueryState.FINISHED, "executor-query-id");
                client.scheduledQueryProgress(info);
                ScheduledQuery schq = client.getScheduledQuery(pollResult.getScheduleKey());
                assertTrue("Scheduled query q2 must be enabled", schq.isEnabled());
            }
        }
        Thread.sleep(100);
    }
    if (idx1 != q1NumberOfExecutions) {
        fail("expected " + q1NumberOfExecutions + " execution of q1; only " + idx1 + " happened");
    }
    if (idx2 < q2NumberOfExecutions) {
        fail("at least " + q2NumberOfExecutions + " expected for q2");
    }
}

Example 20 with ScheduledQuery

use of org.apache.hadoop.hive.metastore.api.ScheduledQuery in project hive by apache.

the class TestMetastoreScheduledQueries method createEverySecondSchq.

private void createEverySecondSchq(ScheduledQueryKey schqKey) throws MetaException, TException {
    ScheduledQuery schq = createScheduledQuery(schqKey);
    ScheduledQueryMaintenanceRequest r = new ScheduledQueryMaintenanceRequest();
    r.setType(ScheduledQueryMaintenanceRequestType.CREATE);
    r.setScheduledQuery(schq);
    client.scheduledQueryMaintenance(r);
}