Examples with ScheduledQuery org.apache.hadoop.hive.metastore.api.ScheduledQuery used on opensource projects

Example 11 with ScheduledQuery

use of org.apache.hadoop.hive.metastore.api.ScheduledQuery in project hive by apache.

the class TestMetastoreScheduledQueries method testCreateWithInvalidSchedule.

@Test(expected = InvalidInputException.class)
public void testCreateWithInvalidSchedule() throws Exception {
    ScheduledQuery schq = createScheduledQuery(createKey("createInvalidSch", "c1"));
    schq.setSchedule("asd asd");
    ScheduledQueryMaintenanceRequest r = new ScheduledQueryMaintenanceRequest();
    r.setType(ScheduledQueryMaintenanceRequestType.CREATE);
    r.setScheduledQuery(schq);
    client.scheduledQueryMaintenance(r);
}

Example 12 with ScheduledQuery

use of org.apache.hadoop.hive.metastore.api.ScheduledQuery in project hive by apache.

the class TestMetastoreScheduledQueries method testOutdatedCleanup.

@Test
public void testOutdatedCleanup() throws Exception {
    String namespace = "outdatedcleanup";
    ObjectStore objStore = new ObjectStore();
    objStore.setConf(metaStore.getConf());
    objStore.deleteScheduledExecutions(0);
    ScheduledQuery schq = createScheduledQuery(new ScheduledQueryKey("q1", namespace));
    ScheduledQueryMaintenanceRequest r = new ScheduledQueryMaintenanceRequest();
    r.setType(ScheduledQueryMaintenanceRequestType.CREATE);
    r.setScheduledQuery(schq);
    objStore.scheduledQueryMaintenance(r);
    Thread.sleep(1000);
    ScheduledQueryPollRequest request = new ScheduledQueryPollRequest(namespace);
    ScheduledQueryPollResponse pollResult = objStore.scheduledQueryPoll(request);
    // will add q1 as a query being executed
    Thread.sleep(1000);
    objStore.markScheduledExecutionsTimedOut(0);
    try (PersistenceManager pm = PersistenceManagerProvider.getPersistenceManager()) {
        MScheduledExecution execution = pm.getObjectById(MScheduledExecution.class, pollResult.getExecutionId());
        assertEquals(QueryState.TIMED_OUT, execution.getState());
    }
}

Example 13 with ScheduledQuery

use of org.apache.hadoop.hive.metastore.api.ScheduledQuery in project hive by apache.

the class ObjectStore method processScheduledQueryPolicies.

private void processScheduledQueryPolicies(ScheduledQueryProgressInfo info) throws MetaException {
    if (info.getState() != QueryState.FAILED && info.getState() != QueryState.TIMED_OUT) {
        return;
    }
    int autoDisableCount = MetastoreConf.getIntVar(conf, ConfVars.SCHEDULED_QUERIES_AUTODISABLE_COUNT);
    int skipCount = MetastoreConf.getIntVar(conf, ConfVars.SCHEDULED_QUERIES_SKIP_OPPORTUNITIES_AFTER_FAILURES);
    int lastN = Math.max(autoDisableCount, skipCount);
    if (lastN <= 0) {
        // disabled
        return;
    }
    boolean commited = false;
    Query query = null;
    try {
        openTransaction();
        MScheduledExecution lastExecution = pm.getObjectById(MScheduledExecution.class, info.getScheduledExecutionId());
        MScheduledQuery schq = lastExecution.getScheduledQuery();
        query = pm.newQuery(MScheduledExecution.class);
        query.setFilter("scheduledQuery == currentSchedule");
        query.setOrdering("scheduledExecutionId descending");
        query.declareParameters("MScheduledQuery currentSchedule");
        query.setRange(0, lastN);
        List<MScheduledExecution> list = (List<MScheduledExecution>) query.execute(schq);
        int failureCount = 0;
        for (int i = 0; i < list.size(); i++) {
            if (list.get(i).getState() != QueryState.FAILED && list.get(i).getState() != QueryState.TIMED_OUT) {
                break;
            }
            failureCount++;
        }
        if (autoDisableCount > 0 && autoDisableCount <= failureCount) {
            LOG.info("Disabling {} after {} consequtive failures", schq.getScheduleKey(), autoDisableCount);
            schq.setEnabled(false);
            int now = (int) (System.currentTimeMillis() / 1000);
            MScheduledExecution execution = new MScheduledExecution();
            execution.setScheduledQuery(schq);
            execution.setState(QueryState.AUTO_DISABLED);
            execution.setStartTime(now);
            execution.setEndTime(now);
            execution.setLastUpdateTime(now);
            execution.setErrorMessage(String.format("Disabling query after {} consequtive failures", autoDisableCount));
            pm.makePersistent(execution);
        }
        if (skipCount > 0) {
            int n = Math.min(skipCount, failureCount) - 1;
            Integer scheduledTime = schq.getNextExecution();
            for (int i = 0; i < n; i++) {
                if (scheduledTime != null) {
                    scheduledTime = computeNextExecutionTime(schq.getSchedule(), scheduledTime);
                }
            }
            if (scheduledTime != null) {
                schq.setNextExecution(scheduledTime);
            }
        }
        commited = commitTransaction();
    } catch (InvalidInputException e) {
        throw new MetaException("Unexpected InvalidInputException: " + e.getMessage());
    } finally {
        rollbackAndCleanup(commited, query);
    }
}

Example 14 with ScheduledQuery

use of org.apache.hadoop.hive.metastore.api.ScheduledQuery in project hive by apache.

the class MScheduledQuery method toThrift.

public static ScheduledQuery toThrift(MScheduledQuery s) {
    ScheduledQuery ret = new ScheduledQuery();
    ret.setScheduleKey(new ScheduledQueryKey(s.scheduleName, s.clusterNamespace));
    ret.setEnabled(s.enabled);
    ret.setSchedule(s.schedule);
    ret.setUser(s.user);
    ret.setQuery(s.query);
    ret.setNextExecution(s.nextExecution);
    return ret;
}

Example 15 with ScheduledQuery

use of org.apache.hadoop.hive.metastore.api.ScheduledQuery in project hive by apache.

the class ScheduledQueryAnalyzer method buildEmptySchq.

private ScheduledQuery buildEmptySchq() {
    ScheduledQuery ret = new ScheduledQuery();
    ret.setEnabled(conf.getBoolVar(ConfVars.HIVE_SCHEDULED_QUERIES_CREATE_AS_ENABLED));
    ret.setUser(getUserName());
    return ret;
}