Search in sources :

Example 1 with LockModel

use of org.opensearch.jobscheduler.spi.LockModel in project anomaly-detection by opensearch-project.

the class AnomalyDetectorJobRunner method runAnomalyDetectionJob.

private void runAnomalyDetectionJob(AnomalyDetectorJob jobParameter, LockService lockService, LockModel lock, Instant detectionStartTime, Instant executionStartTime, String detectorId, String user, List<String> roles) {
    try (InjectSecurity injectSecurity = new InjectSecurity(detectorId, settings, client.threadPool().getThreadContext())) {
        // Injecting user role to verify if the user has permissions for our API.
        injectSecurity.inject(user, roles);
        AnomalyResultRequest request = new AnomalyResultRequest(detectorId, detectionStartTime.toEpochMilli(), executionStartTime.toEpochMilli());
        client.execute(AnomalyResultAction.INSTANCE, request, ActionListener.wrap(response -> {
            indexAnomalyResult(jobParameter, lockService, lock, detectionStartTime, executionStartTime, response);
        }, exception -> {
            handleAdException(jobParameter, lockService, lock, detectionStartTime, executionStartTime, exception);
        }));
    } catch (Exception e) {
        indexAnomalyResultException(jobParameter, lockService, lock, detectionStartTime, executionStartTime, e, true);
        log.error("Failed to execute AD job " + detectorId, e);
    }
}
Also used : ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) ADTaskState(org.opensearch.ad.model.ADTaskState) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) AnomalyDetectorFunction(org.opensearch.ad.rest.handler.AnomalyDetectorFunction) XContentParser(org.opensearch.common.xcontent.XContentParser) AD_THREAD_POOL_NAME(org.opensearch.ad.AnomalyDetectorPlugin.AD_THREAD_POOL_NAME) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) WriteRequest(org.opensearch.action.support.WriteRequest) ADIndex(org.opensearch.ad.indices.ADIndex) ActionListener(org.opensearch.action.ActionListener) GetResponse(org.opensearch.action.get.GetResponse) Client(org.opensearch.client.Client) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) LoggingDeprecationHandler(org.opensearch.common.xcontent.LoggingDeprecationHandler) Set(java.util.Set) XContentParserUtils.ensureExpectedToken(org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken) ADTaskManager(org.opensearch.ad.task.ADTaskManager) Settings(org.opensearch.common.settings.Settings) Instant(java.time.Instant) UPDATED(org.opensearch.action.DocWriteResponse.Result.UPDATED) List(java.util.List) AnomalyResultAction(org.opensearch.ad.transport.AnomalyResultAction) Logger(org.apache.logging.log4j.Logger) AnomalyResult(org.opensearch.ad.model.AnomalyResult) CAN_NOT_FIND_LATEST_TASK(org.opensearch.ad.constant.CommonErrorMessages.CAN_NOT_FIND_LATEST_TASK) CREATED(org.opensearch.action.DocWriteResponse.Result.CREATED) IntervalSchedule(org.opensearch.jobscheduler.spi.schedule.IntervalSchedule) XContentType(org.opensearch.common.xcontent.XContentType) InjectSecurity(org.opensearch.commons.InjectSecurity) JobExecutionContext(org.opensearch.jobscheduler.spi.JobExecutionContext) DiscoveryNodeFilterer(org.opensearch.ad.util.DiscoveryNodeFilterer) AnomalyIndexHandler(org.opensearch.ad.transport.handler.AnomalyIndexHandler) AnomalyResultRequest(org.opensearch.ad.transport.AnomalyResultRequest) ThreadPool(org.opensearch.threadpool.ThreadPool) FeatureData(org.opensearch.ad.model.FeatureData) AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) ArrayList(java.util.ArrayList) ProfileRequest(org.opensearch.ad.transport.ProfileRequest) XCONTENT_WITH_TYPE(org.opensearch.ad.util.RestHandlerUtils.XCONTENT_WITH_TYPE) HashSet(java.util.HashSet) ImmutableList(com.google.common.collect.ImmutableList) LockModel(org.opensearch.jobscheduler.spi.LockModel) ExecutorService(java.util.concurrent.ExecutorService) EndRunException(org.opensearch.ad.common.exception.EndRunException) LockService(org.opensearch.jobscheduler.spi.utils.LockService) InternalFailure(org.opensearch.ad.common.exception.InternalFailure) GetRequest(org.opensearch.action.get.GetRequest) Throwables(com.google.common.base.Throwables) IOException(java.io.IOException) AnomalyDetectionIndices(org.opensearch.ad.indices.AnomalyDetectionIndices) DetectorProfileName(org.opensearch.ad.model.DetectorProfileName) ScheduledJobParameter(org.opensearch.jobscheduler.spi.ScheduledJobParameter) AnomalyResultTransportAction(org.opensearch.ad.transport.AnomalyResultTransportAction) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) User(org.opensearch.commons.authuser.User) AnomalyResultResponse(org.opensearch.ad.transport.AnomalyResultResponse) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ScheduledJobRunner(org.opensearch.jobscheduler.spi.ScheduledJobRunner) IndexRequest(org.opensearch.action.index.IndexRequest) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) LogManager(org.apache.logging.log4j.LogManager) ProfileAction(org.opensearch.ad.transport.ProfileAction) AnomalyResultRequest(org.opensearch.ad.transport.AnomalyResultRequest) InjectSecurity(org.opensearch.commons.InjectSecurity) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) EndRunException(org.opensearch.ad.common.exception.EndRunException) IOException(java.io.IOException)

Example 2 with LockModel

use of org.opensearch.jobscheduler.spi.LockModel in project anomaly-detection by opensearch-project.

the class AnomalyDetectorJobRunnerTests method testRunAdJobWithLock.

@Test
public void testRunAdJobWithLock() {
    LockModel lock = new LockModel("indexName", "jobId", Instant.now(), 10, false);
    runner.runAdJob(jobParameter, lockService, lock, Instant.now().minusMillis(1000 * 60), Instant.now());
    verify(client, times(1)).execute(any(), any(), any());
}
Also used : LockModel(org.opensearch.jobscheduler.spi.LockModel) Test(org.junit.Test)

Example 3 with LockModel

use of org.opensearch.jobscheduler.spi.LockModel in project anomaly-detection by opensearch-project.

the class AnomalyDetectorJobRunnerTests method testRunAdJobWithEndRunExceptionNotNowAndRetryUntilStop.

@Test
public void testRunAdJobWithEndRunExceptionNotNowAndRetryUntilStop() throws InterruptedException {
    LockModel lock = new LockModel(AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX, jobParameter.getName(), Instant.now(), 10, false);
    Instant executionStartTime = Instant.now();
    Schedule schedule = mock(IntervalSchedule.class);
    when(jobParameter.getSchedule()).thenReturn(schedule);
    when(schedule.getNextExecutionTime(executionStartTime)).thenReturn(executionStartTime.plusSeconds(5));
    doAnswer(invocation -> {
        Exception exception = new EndRunException(jobParameter.getName(), randomAlphaOfLength(5), false);
        ActionListener<?> listener = invocation.getArgument(2);
        listener.onFailure(exception);
        return null;
    }).when(client).execute(any(), any(), any());
    for (int i = 0; i < 3; i++) {
        runner.runAdJob(jobParameter, lockService, lock, Instant.now().minusSeconds(60), executionStartTime);
        assertEquals(i + 1, testAppender.countMessage("EndRunException happened for"));
    }
    runner.runAdJob(jobParameter, lockService, lock, Instant.now().minusSeconds(60), executionStartTime);
    assertEquals(1, testAppender.countMessage("JobRunner will stop AD job due to EndRunException retry exceeds upper limit"));
}
Also used : EndRunException(org.opensearch.ad.common.exception.EndRunException) Instant(java.time.Instant) IntervalSchedule(org.opensearch.jobscheduler.spi.schedule.IntervalSchedule) Schedule(org.opensearch.jobscheduler.spi.schedule.Schedule) LockModel(org.opensearch.jobscheduler.spi.LockModel) ExpectedException(org.junit.rules.ExpectedException) EndRunException(org.opensearch.ad.common.exception.EndRunException) Test(org.junit.Test)

Example 4 with LockModel

use of org.opensearch.jobscheduler.spi.LockModel in project anomaly-detection by opensearch-project.

the class AnomalyDetectorJobRunnerTests method testRunAdJobWithNullLock.

@Test
public void testRunAdJobWithNullLock() {
    LockModel lock = null;
    runner.runAdJob(jobParameter, lockService, lock, Instant.now().minusMillis(1000 * 60), Instant.now());
    verify(client, never()).execute(any(), any(), any());
}
Also used : LockModel(org.opensearch.jobscheduler.spi.LockModel) Test(org.junit.Test)

Example 5 with LockModel

use of org.opensearch.jobscheduler.spi.LockModel in project job-scheduler by opensearch-project.

the class LockService method acquireLock.

/**
 * Attempts to acquire lock the job. If the lock does not exists it attempts to create the lock document.
 * If the Lock document exists, it will try to update and acquire lock.
 *
 * @param jobParameter a {@code ScheduledJobParameter} containing the lock duration.
 * @param context a {@code JobExecutionContext} containing job index name and job id.
 * @param listener an {@code ActionListener} that has onResponse and onFailure that is used to return the lock if it was acquired
 *                 or else null. Passes {@code IllegalArgumentException} to onFailure if the {@code ScheduledJobParameter} does not
 *                 have {@code LockDurationSeconds}.
 */
public void acquireLock(final ScheduledJobParameter jobParameter, final JobExecutionContext context, ActionListener<LockModel> listener) {
    final String jobIndexName = context.getJobIndexName();
    final String jobId = context.getJobId();
    if (jobParameter.getLockDurationSeconds() == null) {
        listener.onFailure(new IllegalArgumentException("Job LockDuration should not be null"));
    } else {
        final long lockDurationSecond = jobParameter.getLockDurationSeconds();
        createLockIndex(ActionListener.wrap(created -> {
            if (created) {
                try {
                    findLock(LockModel.generateLockId(jobIndexName, jobId), ActionListener.wrap(existingLock -> {
                        if (existingLock != null) {
                            if (isLockReleasedOrExpired(existingLock)) {
                                // Lock is expired. Attempt to acquire lock.
                                logger.debug("lock is released or expired: " + existingLock);
                                LockModel updateLock = new LockModel(existingLock, getNow(), lockDurationSecond, false);
                                updateLock(updateLock, listener);
                            } else {
                                logger.debug("Lock is NOT released or expired. " + existingLock);
                                // Lock is still not expired. Return null as we cannot acquire lock.
                                listener.onResponse(null);
                            }
                        } else {
                            // There is no lock object and it is first time. Create new lock.
                            LockModel tempLock = new LockModel(jobIndexName, jobId, getNow(), lockDurationSecond, false);
                            logger.debug("Lock does not exist. Creating new lock" + tempLock);
                            createLock(tempLock, listener);
                        }
                    }, listener::onFailure));
                } catch (VersionConflictEngineException e) {
                    logger.debug("could not acquire lock {}", e.getMessage());
                    listener.onResponse(null);
                }
            } else {
                listener.onResponse(null);
            }
        }, listener::onFailure));
    }
}
Also used : SequenceNumbers(org.opensearch.index.seqno.SequenceNumbers) ToXContent(org.opensearch.common.xcontent.ToXContent) XContentParser(org.opensearch.common.xcontent.XContentParser) XContentFactory(org.opensearch.common.xcontent.XContentFactory) ActionListener(org.opensearch.action.ActionListener) LockModel(org.opensearch.jobscheduler.spi.LockModel) CreateIndexRequest(org.opensearch.action.admin.indices.create.CreateIndexRequest) DeleteRequest(org.opensearch.action.delete.DeleteRequest) Client(org.opensearch.client.Client) VersionConflictEngineException(org.opensearch.index.engine.VersionConflictEngineException) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) GetRequest(org.opensearch.action.get.GetRequest) LoggingDeprecationHandler(org.opensearch.common.xcontent.LoggingDeprecationHandler) IOException(java.io.IOException) DocumentMissingException(org.opensearch.index.engine.DocumentMissingException) Instant(java.time.Instant) ScheduledJobParameter(org.opensearch.jobscheduler.spi.ScheduledJobParameter) InputStreamReader(java.io.InputStreamReader) StandardCharsets(java.nio.charset.StandardCharsets) Logger(org.apache.logging.log4j.Logger) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) VisibleForTesting(com.cronutils.utils.VisibleForTesting) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) DocWriteResponse(org.opensearch.action.DocWriteResponse) UpdateRequest(org.opensearch.action.update.UpdateRequest) XContentType(org.opensearch.common.xcontent.XContentType) JobExecutionContext(org.opensearch.jobscheduler.spi.JobExecutionContext) BufferedReader(java.io.BufferedReader) IndexRequest(org.opensearch.action.index.IndexRequest) LogManager(org.apache.logging.log4j.LogManager) InputStream(java.io.InputStream) VersionConflictEngineException(org.opensearch.index.engine.VersionConflictEngineException) LockModel(org.opensearch.jobscheduler.spi.LockModel)

Aggregations

LockModel (org.opensearch.jobscheduler.spi.LockModel)18 Instant (java.time.Instant)11 ActionListener (org.opensearch.action.ActionListener)11 GetRequest (org.opensearch.action.get.GetRequest)11 IOException (java.io.IOException)10 JobExecutionContext (org.opensearch.jobscheduler.spi.JobExecutionContext)10 ScheduledJobParameter (org.opensearch.jobscheduler.spi.ScheduledJobParameter)10 IndexRequest (org.opensearch.action.index.IndexRequest)9 ClusterService (org.opensearch.cluster.service.ClusterService)9 LogManager (org.apache.logging.log4j.LogManager)8 Logger (org.apache.logging.log4j.Logger)8 Client (org.opensearch.client.Client)8 LoggingDeprecationHandler (org.opensearch.common.xcontent.LoggingDeprecationHandler)8 NamedXContentRegistry (org.opensearch.common.xcontent.NamedXContentRegistry)8 XContentParser (org.opensearch.common.xcontent.XContentParser)8 XContentType (org.opensearch.common.xcontent.XContentType)8 VisibleForTesting (com.cronutils.utils.VisibleForTesting)7 BufferedReader (java.io.BufferedReader)7 InputStream (java.io.InputStream)7 InputStreamReader (java.io.InputStreamReader)7