Examples with MScheduledExecution org.apache.hadoop.hive.metastore.model.MScheduledExecution used on opensource projects

Example 1 with MScheduledExecution

use of org.apache.hadoop.hive.metastore.model.MScheduledExecution in project hive by apache.

the class TestMetastoreScheduledQueries method testCleanup.

@Test
public void testCleanup() throws Exception {
    String namespace = "cleanup";
    ObjectStore objStore = new ObjectStore();
    objStore.setConf(metaStore.getConf());
    objStore.deleteScheduledExecutions(0);
    ScheduledQuery schq = createScheduledQuery(new ScheduledQueryKey("q1", namespace));
    ScheduledQueryMaintenanceRequest r = new ScheduledQueryMaintenanceRequest();
    r.setType(ScheduledQueryMaintenanceRequestType.CREATE);
    r.setScheduledQuery(schq);
    objStore.scheduledQueryMaintenance(r);
    Thread.sleep(1000);
    ScheduledQueryPollRequest request = new ScheduledQueryPollRequest(namespace);
    ScheduledQueryPollResponse pollResult = objStore.scheduledQueryPoll(request);
    try (PersistenceManager pm = PersistenceManagerProvider.getPersistenceManager()) {
        MScheduledExecution q = pm.getObjectById(MScheduledExecution.class, pollResult.getExecutionId());
        assertEquals(QueryState.INITED, q.getState());
    }
    Thread.sleep(1000);
    objStore.deleteScheduledExecutions(0);
    try (PersistenceManager pm = PersistenceManagerProvider.getPersistenceManager()) {
        try {
            pm.getObjectById(MScheduledExecution.class, pollResult.getExecutionId());
            fail("The execution is expected to be deleted at this point...");
        } catch (JDOObjectNotFoundException e) {
        // expected
        }
    }
}

Example 2 with MScheduledExecution

use of org.apache.hadoop.hive.metastore.model.MScheduledExecution in project hive by apache.

the class TestMetastoreScheduledQueries method testOutdatedCleanup.

@Test
public void testOutdatedCleanup() throws Exception {
    String namespace = "outdatedcleanup";
    ObjectStore objStore = new ObjectStore();
    objStore.setConf(metaStore.getConf());
    objStore.deleteScheduledExecutions(0);
    ScheduledQuery schq = createScheduledQuery(new ScheduledQueryKey("q1", namespace));
    ScheduledQueryMaintenanceRequest r = new ScheduledQueryMaintenanceRequest();
    r.setType(ScheduledQueryMaintenanceRequestType.CREATE);
    r.setScheduledQuery(schq);
    objStore.scheduledQueryMaintenance(r);
    Thread.sleep(1000);
    ScheduledQueryPollRequest request = new ScheduledQueryPollRequest(namespace);
    ScheduledQueryPollResponse pollResult = objStore.scheduledQueryPoll(request);
    // will add q1 as a query being executed
    Thread.sleep(1000);
    objStore.markScheduledExecutionsTimedOut(0);
    try (PersistenceManager pm = PersistenceManagerProvider.getPersistenceManager()) {
        MScheduledExecution execution = pm.getObjectById(MScheduledExecution.class, pollResult.getExecutionId());
        assertEquals(QueryState.TIMED_OUT, execution.getState());
    }
}

Example 3 with MScheduledExecution

use of org.apache.hadoop.hive.metastore.model.MScheduledExecution in project hive by apache.

the class ObjectStore method processScheduledQueryPolicies.

private void processScheduledQueryPolicies(ScheduledQueryProgressInfo info) throws MetaException {
    if (info.getState() != QueryState.FAILED && info.getState() != QueryState.TIMED_OUT) {
        return;
    }
    int autoDisableCount = MetastoreConf.getIntVar(conf, ConfVars.SCHEDULED_QUERIES_AUTODISABLE_COUNT);
    int skipCount = MetastoreConf.getIntVar(conf, ConfVars.SCHEDULED_QUERIES_SKIP_OPPORTUNITIES_AFTER_FAILURES);
    int lastN = Math.max(autoDisableCount, skipCount);
    if (lastN <= 0) {
        // disabled
        return;
    }
    boolean commited = false;
    Query query = null;
    try {
        openTransaction();
        MScheduledExecution lastExecution = pm.getObjectById(MScheduledExecution.class, info.getScheduledExecutionId());
        MScheduledQuery schq = lastExecution.getScheduledQuery();
        query = pm.newQuery(MScheduledExecution.class);
        query.setFilter("scheduledQuery == currentSchedule");
        query.setOrdering("scheduledExecutionId descending");
        query.declareParameters("MScheduledQuery currentSchedule");
        query.setRange(0, lastN);
        List<MScheduledExecution> list = (List<MScheduledExecution>) query.execute(schq);
        int failureCount = 0;
        for (int i = 0; i < list.size(); i++) {
            if (list.get(i).getState() != QueryState.FAILED && list.get(i).getState() != QueryState.TIMED_OUT) {
                break;
            }
            failureCount++;
        }
        if (autoDisableCount > 0 && autoDisableCount <= failureCount) {
            LOG.info("Disabling {} after {} consequtive failures", schq.getScheduleKey(), autoDisableCount);
            schq.setEnabled(false);
            int now = (int) (System.currentTimeMillis() / 1000);
            MScheduledExecution execution = new MScheduledExecution();
            execution.setScheduledQuery(schq);
            execution.setState(QueryState.AUTO_DISABLED);
            execution.setStartTime(now);
            execution.setEndTime(now);
            execution.setLastUpdateTime(now);
            execution.setErrorMessage(String.format("Disabling query after {} consequtive failures", autoDisableCount));
            pm.makePersistent(execution);
        }
        if (skipCount > 0) {
            int n = Math.min(skipCount, failureCount) - 1;
            Integer scheduledTime = schq.getNextExecution();
            for (int i = 0; i < n; i++) {
                if (scheduledTime != null) {
                    scheduledTime = computeNextExecutionTime(schq.getSchedule(), scheduledTime);
                }
            }
            if (scheduledTime != null) {
                schq.setNextExecution(scheduledTime);
            }
        }
        commited = commitTransaction();
    } catch (InvalidInputException e) {
        throw new MetaException("Unexpected InvalidInputException: " + e.getMessage());
    } finally {
        rollbackAndCleanup(commited, query);
    }
}

Example 4 with MScheduledExecution

use of org.apache.hadoop.hive.metastore.model.MScheduledExecution in project hive by apache.

the class ObjectStore method scheduledQueryPoll.

@Override
public ScheduledQueryPollResponse scheduledQueryPoll(ScheduledQueryPollRequest request) throws MetaException {
    ensureScheduledQueriesEnabled();
    boolean commited = false;
    ScheduledQueryPollResponse ret = new ScheduledQueryPollResponse();
    Query q = null;
    try {
        openTransaction();
        q = pm.newQuery(MScheduledQuery.class, "nextExecution <= now && enabled && clusterNamespace == ns && activeExecution == null");
        q.setSerializeRead(true);
        q.declareParameters("java.lang.Integer now, java.lang.String ns");
        q.setOrdering("nextExecution");
        int now = (int) (System.currentTimeMillis() / 1000);
        List<MScheduledQuery> results = (List<MScheduledQuery>) q.execute(now, request.getClusterNamespace());
        if (results == null || results.isEmpty()) {
            return new ScheduledQueryPollResponse();
        }
        MScheduledQuery schq = results.get(0);
        schq.setNextExecution(computeNextExecutionTime(schq.getSchedule()));
        MScheduledExecution execution = new MScheduledExecution();
        execution.setScheduledQuery(schq);
        execution.setState(QueryState.INITED);
        execution.setStartTime(now);
        execution.setLastUpdateTime(now);
        schq.setActiveExecution(execution);
        pm.makePersistent(execution);
        pm.makePersistent(schq);
        ObjectStoreTestHook.onScheduledQueryPoll();
        commited = commitTransaction();
        ret.setScheduleKey(schq.getScheduleKey());
        ret.setQuery("/* schedule: " + schq.getScheduleName() + " */" + schq.getQuery());
        ret.setUser(schq.getUser());
        int executionId = ((IntIdentity) pm.getObjectId(execution)).getKey();
        ret.setExecutionId(executionId);
    } catch (JDOException e) {
        LOG.debug("Caught jdo exception; exclusive", e);
        commited = false;
    } finally {
        rollbackAndCleanup(commited, q);
        return commited ? ret : new ScheduledQueryPollResponse();
    }
}

Example 5 with MScheduledExecution

use of org.apache.hadoop.hive.metastore.model.MScheduledExecution in project hive by apache.

the class ObjectStore method scheduledQueryProgress.

@Override
public void scheduledQueryProgress(ScheduledQueryProgressInfo info) throws InvalidOperationException, MetaException {
    ensureScheduledQueriesEnabled();
    boolean commited = false;
    try {
        openTransaction();
        MScheduledExecution execution = pm.getObjectById(MScheduledExecution.class, info.getScheduledExecutionId());
        if (!validateStateChange(execution.getState(), info.getState())) {
            throw new InvalidOperationException("Invalid state change: " + execution.getState() + "=>" + info.getState());
        }
        execution.setState(info.getState());
        if (info.isSetExecutorQueryId()) {
            execution.setExecutorQueryId(info.getExecutorQueryId());
        }
        if (info.isSetErrorMessage()) {
            execution.setErrorMessage(abbreviateErrorMessage(info.getErrorMessage(), 1000));
        }
        switch(info.getState()) {
            case INITED:
            case EXECUTING:
                execution.setLastUpdateTime((int) (System.currentTimeMillis() / 1000));
                break;
            case FAILED:
            case FINISHED:
            case TIMED_OUT:
                execution.setEndTime((int) (System.currentTimeMillis() / 1000));
                execution.setLastUpdateTime(null);
                execution.getScheduledQuery().setActiveExecution(null);
                break;
            default:
                throw new InvalidOperationException("invalid state: " + info.getState());
        }
        pm.makePersistent(execution);
        processScheduledQueryPolicies(info);
        commited = commitTransaction();
    } finally {
        if (!commited) {
            rollbackTransaction();
        }
    }
}