use of org.graylog.events.TestEventProcessorParameters in project graylog2-server by Graylog2.
the class EventProcessorExecutionJobTest method catchupWindowTestHelper.
private void catchupWindowTestHelper(long catchUpWindowSize, long processingHopSize, long processingWindowSize) throws Exception {
when(eventsConfigurationProvider.get()).thenReturn(EventsConfiguration.builder().eventCatchupWindow(catchUpWindowSize).build());
// for easier testing. don't run into the previous day
clock.plus(1, TimeUnit.MINUTES);
final DateTime now = clock.nowUTC();
final long processingCatchUpWindowSize = eventsConfigurationProvider.get().eventCatchupWindow();
final int scheduleIntervalSeconds = (int) processingHopSize * 1000;
final DateTime from = now.minus(processingWindowSize);
final DateTime to = now;
final DateTime triggerNextTime = now;
final Duration timeSpentInEventProcessor = Duration.standardSeconds(7);
final TestEventProcessorParameters eventProcessorParameters = TestEventProcessorParameters.create(from, to);
final JobDefinitionDto jobDefinition = JobDefinitionDto.builder().id("job-1").title("Test").description("A test").config(EventProcessorExecutionJob.Config.builder().eventDefinitionId("processor-1").processingWindowSize(processingWindowSize).processingHopSize(processingHopSize).parameters(eventProcessorParameters).build()).build();
final EventProcessorExecutionJob job = new EventProcessorExecutionJob(jobScheduleStrategies, clock, eventProcessorEngine, eventsConfigurationProvider, jobDefinition);
final JobTriggerDto trigger = JobTriggerDto.builderWithClock(clock).id("trigger-1").jobDefinitionId(jobDefinition.id()).startTime(now).nextTime(triggerNextTime).status(JobTriggerStatus.RUNNABLE).schedule(IntervalJobSchedule.builder().interval(scheduleIntervalSeconds).unit(TimeUnit.SECONDS).build()).build();
final JobExecutionContext jobExecutionContext = JobExecutionContext.builder().definition(jobDefinition).trigger(trigger).isRunning(new AtomicBoolean(true)).jobTriggerUpdates(new JobTriggerUpdates(clock, jobScheduleStrategies, trigger)).build();
doAnswer(invocation -> {
// Simulate work in the event processor
clock.plus(timeSpentInEventProcessor.getStandardSeconds(), TimeUnit.SECONDS);
return null;
}).when(eventProcessorEngine).execute(any(), any());
// Simulate that we are behind at least one `processingCatchUpWindowSize`
clock.plus(EventsConfiguration.DEFAULT_CATCH_UP_WINDOW_MS, TimeUnit.MILLISECONDS);
clock.plus(1, TimeUnit.MILLISECONDS);
final JobTriggerUpdate triggerUpdate = job.execute(jobExecutionContext);
verify(eventProcessorEngine, times(1)).execute("processor-1", eventProcessorParameters);
assertThat(triggerUpdate.nextTime()).isPresent().get().isEqualTo(clock.nowUTC().minus(timeSpentInEventProcessor));
if (catchUpWindowSize > processingWindowSize && processingHopSize <= processingWindowSize) {
// We are behind at least one chunk of catchUpWindowSize
// The new nextFrom should ignore the processingHopSize and start 1ms after the last `To` Range
// The nextTo will be one window of the processingCatchUpWindowSize
assertThat(triggerUpdate.data()).isPresent().get().isEqualTo(EventProcessorExecutionJob.Data.builder().timerangeFrom(to.plus(processingHopSize).minus(processingWindowSize)).timerangeTo(to.plus(processingCatchUpWindowSize)).build());
} else {
// If no catchup is in effect, we fall back into the configured hopping window mode.
// With a hopping window the "to" is calculated by adding the hopSize and the "from" is based on the next "to"
// minus the windowSize + 1 millisecond.
assertThat(triggerUpdate.data()).isPresent().get().isEqualTo(EventProcessorExecutionJob.Data.builder().timerangeFrom(to.plus(processingHopSize).minus(processingWindowSize)).timerangeTo(to.plus(processingHopSize)).build());
}
assertThat(triggerUpdate.status()).isNotPresent();
}
use of org.graylog.events.TestEventProcessorParameters in project graylog2-server by Graylog2.
the class EventProcessorExecutionJobTest method executeWithInvalidTimerange.
@Test
public void executeWithInvalidTimerange() throws Exception {
final DateTime now = clock.nowUTC();
final long processingWindowSize = Duration.standardSeconds(60).getMillis();
final long processingHopSize = Duration.standardSeconds(60).getMillis();
final int scheduleIntervalSeconds = 1;
// We set "from" to be AFTER "to" - this is not valid so the job should not be executed and the triggers
// should be set to ERROR
final DateTime from = now.plusSeconds(1);
final DateTime to = now;
final DateTime triggerNextTime = now;
final TestEventProcessorParameters eventProcessorParameters = TestEventProcessorParameters.create(from, to);
final JobDefinitionDto jobDefinition = JobDefinitionDto.builder().id("job-1").title("Test").description("A test").config(EventProcessorExecutionJob.Config.builder().eventDefinitionId("processor-1").processingWindowSize(processingWindowSize).processingHopSize(processingHopSize).parameters(eventProcessorParameters).build()).build();
final EventProcessorExecutionJob job = new EventProcessorExecutionJob(jobScheduleStrategies, clock, eventProcessorEngine, eventsConfigurationProvider, jobDefinition);
final JobTriggerDto trigger = JobTriggerDto.builderWithClock(clock).id("trigger-1").jobDefinitionId(jobDefinition.id()).startTime(now).nextTime(triggerNextTime).status(JobTriggerStatus.RUNNABLE).schedule(IntervalJobSchedule.builder().interval(scheduleIntervalSeconds).unit(TimeUnit.SECONDS).build()).build();
final JobExecutionContext jobExecutionContext = JobExecutionContext.builder().definition(jobDefinition).trigger(trigger).isRunning(new AtomicBoolean(true)).jobTriggerUpdates(new JobTriggerUpdates(clock, jobScheduleStrategies, trigger)).build();
assertThatThrownBy(() -> job.execute(jobExecutionContext)).isInstanceOf(JobExecutionException.class).hasMessageContaining("is not after").satisfies(t -> {
final JobExecutionException e = (JobExecutionException) t;
assertThat(e.getTrigger()).isEqualTo(trigger);
assertThat(e.getUpdate()).satisfies(update -> {
// When setting the status to ERROR, we will keen the last nextTime
assertThat(update.nextTime()).isPresent().get().isEqualTo(triggerNextTime);
assertThat(update.data()).isNotPresent();
assertThat(update.status()).isPresent().get().isEqualTo(JobTriggerStatus.ERROR);
});
});
// The engine should not be called because the timerange is invalid
verify(eventProcessorEngine, never()).execute(any(), any());
}
use of org.graylog.events.TestEventProcessorParameters in project graylog2-server by Graylog2.
the class EventProcessorExecutionJobTest method executeWithNextTimeNotBasedOnCurrentTime.
@Test
public void executeWithNextTimeNotBasedOnCurrentTime() throws Exception {
final DateTime now = clock.nowUTC();
final long processingWindowSize = Duration.standardSeconds(60).getMillis();
final long processingHopSize = Duration.standardSeconds(60).getMillis();
final int scheduleIntervalSeconds = 1;
final DateTime from = now.minus(processingWindowSize);
final DateTime to = now;
final DateTime triggerNextTime = now;
final TestEventProcessorParameters eventProcessorParameters = TestEventProcessorParameters.create(from, to);
final JobDefinitionDto jobDefinition = JobDefinitionDto.builder().id("job-1").title("Test").description("A test").config(EventProcessorExecutionJob.Config.builder().eventDefinitionId("processor-1").processingWindowSize(processingWindowSize).processingHopSize(processingHopSize).parameters(eventProcessorParameters).build()).build();
final EventProcessorExecutionJob job = new EventProcessorExecutionJob(jobScheduleStrategies, clock, eventProcessorEngine, eventsConfigurationProvider, jobDefinition);
final JobTriggerDto trigger = JobTriggerDto.builderWithClock(clock).id("trigger-1").jobDefinitionId(jobDefinition.id()).startTime(now).nextTime(triggerNextTime).status(JobTriggerStatus.RUNNABLE).schedule(IntervalJobSchedule.builder().interval(scheduleIntervalSeconds).unit(TimeUnit.SECONDS).build()).build();
final JobExecutionContext jobExecutionContext = JobExecutionContext.builder().definition(jobDefinition).trigger(trigger).isRunning(new AtomicBoolean(true)).jobTriggerUpdates(new JobTriggerUpdates(clock, jobScheduleStrategies, trigger)).build();
doAnswer(invocation -> {
// Simulate work in the event processor
clock.plus(10, TimeUnit.SECONDS);
return null;
}).when(eventProcessorEngine).execute(any(), any());
final JobTriggerUpdate triggerUpdate = job.execute(jobExecutionContext);
verify(eventProcessorEngine, times(1)).execute("processor-1", eventProcessorParameters);
// The next time should be based on the previous nextTime + the schedule. The 10 second event processor
// runtime should not be added to the new nextTime.
assertThat(triggerUpdate.nextTime()).isPresent().get().isEqualTo(triggerNextTime.plusSeconds(scheduleIntervalSeconds));
assertThat(triggerUpdate.data()).isPresent().get().isEqualTo(EventProcessorExecutionJob.Data.builder().timerangeFrom(to).timerangeTo(to.plus(processingWindowSize)).build());
assertThat(triggerUpdate.status()).isNotPresent();
}
use of org.graylog.events.TestEventProcessorParameters in project graylog2-server by Graylog2.
the class EventProcessorExecutionJobTest method executeWithTriggerDataTimerange.
@Test
public void executeWithTriggerDataTimerange() throws Exception {
final DateTime now = clock.nowUTC();
final long processingWindowSize = Duration.standardSeconds(60).getMillis();
final long processingHopSize = Duration.standardSeconds(60).getMillis();
final int scheduleIntervalSeconds = 1;
final DateTime from = now.minusDays(10).minus(processingWindowSize);
final DateTime to = now.minusDays(10);
final DateTime triggerFrom = now.minus(processingWindowSize);
final DateTime triggerTo = now;
final DateTime triggerNextTime = now;
final TestEventProcessorParameters eventProcessorParameters = TestEventProcessorParameters.create(from, to);
final JobDefinitionDto jobDefinition = JobDefinitionDto.builder().id("job-1").title("Test").description("A test").config(EventProcessorExecutionJob.Config.builder().eventDefinitionId("processor-1").processingWindowSize(processingWindowSize).processingHopSize(processingHopSize).parameters(eventProcessorParameters).build()).build();
final EventProcessorExecutionJob job = new EventProcessorExecutionJob(jobScheduleStrategies, clock, eventProcessorEngine, eventsConfigurationProvider, jobDefinition);
final JobTriggerDto trigger = JobTriggerDto.builderWithClock(clock).id("trigger-1").jobDefinitionId(jobDefinition.id()).startTime(now).nextTime(triggerNextTime).status(JobTriggerStatus.RUNNABLE).schedule(IntervalJobSchedule.builder().interval(scheduleIntervalSeconds).unit(TimeUnit.SECONDS).build()).data(EventProcessorExecutionJob.Data.create(triggerFrom, triggerTo)).build();
final JobExecutionContext jobExecutionContext = JobExecutionContext.builder().definition(jobDefinition).trigger(trigger).isRunning(new AtomicBoolean(true)).jobTriggerUpdates(new JobTriggerUpdates(clock, jobScheduleStrategies, trigger)).build();
doAnswer(invocation -> {
// Simulate work in the event processor
clock.plus(5, TimeUnit.SECONDS);
return null;
}).when(eventProcessorEngine).execute(any(), any());
final JobTriggerUpdate triggerUpdate = job.execute(jobExecutionContext);
// Check that we use the timerange from the trigger instead of the parameters
verify(eventProcessorEngine, times(1)).execute("processor-1", eventProcessorParameters.withTimerange(triggerFrom, triggerTo));
assertThat(triggerUpdate.nextTime()).isPresent().get().isEqualTo(triggerNextTime.plusSeconds(scheduleIntervalSeconds));
// The next timerange in the trigger update also needs to be based on the timerange from the trigger
assertThat(triggerUpdate.data()).isPresent().get().isEqualTo(EventProcessorExecutionJob.Data.builder().timerangeFrom(triggerTo).timerangeTo(triggerTo.plus(processingWindowSize)).build());
assertThat(triggerUpdate.status()).isNotPresent();
}
use of org.graylog.events.TestEventProcessorParameters in project graylog2-server by Graylog2.
the class EventProcessorExecutionJobTest method executeWithTimerangeInTheFuture.
@Test
public void executeWithTimerangeInTheFuture() throws Exception {
final DateTime now = clock.nowUTC();
final long processingWindowSize = Duration.standardSeconds(60).getMillis();
final long processingHopSize = Duration.standardSeconds(60).getMillis();
final int scheduleIntervalSeconds = 1;
final DateTime from = now;
// Set the "to" timestamp of the timerange way into the future
final DateTime to = now.plusDays(1);
final DateTime triggerNextTime = now;
final TestEventProcessorParameters eventProcessorParameters = TestEventProcessorParameters.create(from, to);
final JobDefinitionDto jobDefinition = JobDefinitionDto.builder().id("job-1").title("Test").description("A test").config(EventProcessorExecutionJob.Config.builder().eventDefinitionId("processor-1").processingWindowSize(processingWindowSize).processingHopSize(processingHopSize).parameters(eventProcessorParameters).build()).build();
final EventProcessorExecutionJob job = new EventProcessorExecutionJob(jobScheduleStrategies, clock, eventProcessorEngine, eventsConfigurationProvider, jobDefinition);
final JobTriggerDto trigger = JobTriggerDto.builderWithClock(clock).id("trigger-1").jobDefinitionId(jobDefinition.id()).startTime(now).nextTime(triggerNextTime).status(JobTriggerStatus.RUNNABLE).schedule(IntervalJobSchedule.builder().interval(scheduleIntervalSeconds).unit(TimeUnit.SECONDS).build()).build();
final JobExecutionContext jobExecutionContext = JobExecutionContext.builder().definition(jobDefinition).trigger(trigger).isRunning(new AtomicBoolean(true)).jobTriggerUpdates(new JobTriggerUpdates(clock, jobScheduleStrategies, trigger)).build();
final JobTriggerUpdate triggerUpdate = job.execute(jobExecutionContext);
// The engine should not be called because the "to" timestamp of the timerange is in the future
verify(eventProcessorEngine, never()).execute(any(), any());
// The update sets the nextTime to the "to" timestamp because that is the earliest time we can execute
// the job for the timerange
assertThat(triggerUpdate.nextTime()).isPresent().get().isEqualTo(to);
// Data should not be updated with any new timerange
assertThat(triggerUpdate.data()).isNotPresent();
assertThat(triggerUpdate.status()).isNotPresent();
}
Aggregations