Examples with JobContextLog io.crate.expression.reference.sys.job.JobContextLog used on opensource projects

Example 1 with JobContextLog

use of io.crate.expression.reference.sys.job.JobContextLog in project crate by crate.

the class JobsLogsTest method testExecutionFailureIsRecordedInMetrics.

@Test
public void testExecutionFailureIsRecordedInMetrics() {
    JobsLogs jobsLogs = new JobsLogs(() -> true);
    User user = User.of("arthur");
    Queue<JobContextLog> q = new BlockingEvictingQueue<>(1);
    jobsLogs.updateJobsLog(new QueueSink<>(q, () -> {
    }));
    jobsLogs.logPreExecutionFailure(UUID.randomUUID(), "select foo", "stmt error", user);
    List<MetricsView> metrics = StreamSupport.stream(jobsLogs.metrics().spliterator(), false).collect(Collectors.toList());
    assertThat(metrics.size(), is(1));
    assertThat(metrics.get(0).failedCount(), is(1L));
    assertThat(metrics.get(0).totalCount(), is(1L));
    assertThat(metrics.get(0).classification(), is(new Classification(UNDEFINED)));
}

Example 2 with JobContextLog

use of io.crate.expression.reference.sys.job.JobContextLog in project crate by crate.

the class JobsLogsTest method testReEnableStats.

@Test
public void testReEnableStats() {
    clusterService.getClusterSettings().applySettings(Settings.builder().put(JobsLogService.STATS_ENABLED_SETTING.getKey(), false).build());
    Settings settings = Settings.builder().put(JobsLogService.STATS_ENABLED_SETTING.getKey(), false).put(JobsLogService.STATS_JOBS_LOG_SIZE_SETTING.getKey(), 100).put(JobsLogService.STATS_OPERATIONS_LOG_SIZE_SETTING.getKey(), 100).build();
    JobsLogService stats = new JobsLogService(settings, clusterService::localNode, clusterSettings, nodeCtx, scheduler, breakerService);
    LogSink<JobContextLog> jobsLogSink = (LogSink<JobContextLog>) stats.get().jobsLog();
    LogSink<OperationContextLog> operationsLogSink = (LogSink<OperationContextLog>) stats.get().operationsLog();
    assertThat(stats.isEnabled(), is(false));
    assertThat(stats.jobsLogSize, is(100));
    assertThat(jobsLogSink, Matchers.instanceOf(NoopLogSink.class));
    assertThat(stats.operationsLogSize, is(100));
    assertThat(operationsLogSink, Matchers.instanceOf(NoopLogSink.class));
    clusterService.getClusterSettings().applySettings(Settings.builder().put(JobsLogService.STATS_ENABLED_SETTING.getKey(), true).build());
    assertThat(stats.isEnabled(), is(true));
    assertThat(stats.jobsLogSize, is(100));
    assertThat(stats.get().jobsLog(), Matchers.instanceOf(FilteredLogSink.class));
    assertThat(stats.operationsLogSize, is(100));
    assertThat(stats.get().operationsLog(), Matchers.instanceOf(QueueSink.class));
}

Example 3 with JobContextLog

use of io.crate.expression.reference.sys.job.JobContextLog in project crate by crate.

the class JobsLogsTest method testEntriesCanBeRejectedWithFilter.

@Test
public void testEntriesCanBeRejectedWithFilter() {
    Settings settings = Settings.builder().put(JobsLogService.STATS_JOBS_LOG_FILTER.getKey(), "stmt like 'select%'").build();
    JobsLogService stats = new JobsLogService(settings, clusterService::localNode, clusterSettings, nodeCtx, scheduler, breakerService);
    LogSink<JobContextLog> jobsLogSink = (LogSink<JobContextLog>) stats.get().jobsLog();
    jobsLogSink.add(new JobContextLog(new JobContext(UUID.randomUUID(), "insert into", 10L, User.CRATE_USER, null), null, 20L));
    jobsLogSink.add(new JobContextLog(new JobContext(UUID.randomUUID(), "select * from t1", 10L, User.CRATE_USER, null), null, 20L));
    assertThat(StreamSupport.stream(jobsLogSink.spliterator(), false).count(), is(1L));
}

Example 4 with JobContextLog

use of io.crate.expression.reference.sys.job.JobContextLog in project crate by crate.

the class JobsLogsTest method testSettingsChanges.

@Test
public void testSettingsChanges() throws Exception {
    Settings settings = Settings.builder().put(JobsLogService.STATS_ENABLED_SETTING.getKey(), true).put(JobsLogService.STATS_JOBS_LOG_SIZE_SETTING.getKey(), 100).put(JobsLogService.STATS_OPERATIONS_LOG_SIZE_SETTING.getKey(), 100).build();
    JobsLogService stats = new JobsLogService(settings, clusterService::localNode, clusterSettings, nodeCtx, scheduler, breakerService);
    Supplier<LogSink<JobContextLog>> jobsLogSink = () -> (LogSink<JobContextLog>) stats.get().jobsLog();
    Supplier<LogSink<OperationContextLog>> operationsLogSink = () -> (LogSink<OperationContextLog>) stats.get().operationsLog();
    // sinks are still of type QueueSink
    assertThat(jobsLogSink.get(), Matchers.instanceOf(FilteredLogSink.class));
    assertThat(operationsLogSink.get(), Matchers.instanceOf(QueueSink.class));
    assertThat(inspectRamAccountingQueue((QueueSink) ((FilteredLogSink) jobsLogSink.get()).delegate), Matchers.instanceOf(BlockingEvictingQueue.class));
    assertThat(inspectRamAccountingQueue((QueueSink) operationsLogSink.get()), Matchers.instanceOf(BlockingEvictingQueue.class));
    clusterSettings.applySettings(Settings.builder().put(JobsLogService.STATS_JOBS_LOG_EXPIRATION_SETTING.getKey(), "10s").put(JobsLogService.STATS_OPERATIONS_LOG_EXPIRATION_SETTING.getKey(), "10s").build());
    assertThat(inspectRamAccountingQueue((QueueSink) ((FilteredLogSink<JobContextLog>) jobsLogSink.get()).delegate), Matchers.instanceOf(ConcurrentLinkedDeque.class));
    assertThat(inspectRamAccountingQueue((QueueSink) operationsLogSink.get()), Matchers.instanceOf(ConcurrentLinkedDeque.class));
    // set all to 0 but don't disable stats
    clusterSettings.applySettings(Settings.builder().put(JobsLogService.STATS_JOBS_LOG_SIZE_SETTING.getKey(), 0).put(JobsLogService.STATS_JOBS_LOG_EXPIRATION_SETTING.getKey(), "0s").put(JobsLogService.STATS_OPERATIONS_LOG_SIZE_SETTING.getKey(), 0).put(JobsLogService.STATS_OPERATIONS_LOG_EXPIRATION_SETTING.getKey(), "0s").build());
    assertThat(jobsLogSink.get(), Matchers.instanceOf(NoopLogSink.class));
    assertThat(operationsLogSink.get(), Matchers.instanceOf(NoopLogSink.class));
    assertThat(stats.isEnabled(), is(true));
    clusterSettings.applySettings(Settings.builder().put(JobsLogService.STATS_JOBS_LOG_SIZE_SETTING.getKey(), 200).put(JobsLogService.STATS_OPERATIONS_LOG_SIZE_SETTING.getKey(), 200).put(JobsLogService.STATS_ENABLED_SETTING.getKey(), true).build());
    assertThat(jobsLogSink.get(), Matchers.instanceOf(FilteredLogSink.class));
    assertThat(inspectRamAccountingQueue((QueueSink) ((FilteredLogSink<JobContextLog>) jobsLogSink.get()).delegate), Matchers.instanceOf(BlockingEvictingQueue.class));
    assertThat(operationsLogSink.get(), Matchers.instanceOf(QueueSink.class));
    assertThat(inspectRamAccountingQueue((QueueSink) operationsLogSink.get()), Matchers.instanceOf(BlockingEvictingQueue.class));
    // disable stats
    clusterSettings.applySettings(Settings.builder().put(JobsLogService.STATS_ENABLED_SETTING.getKey(), false).build());
    assertThat(stats.isEnabled(), is(false));
    assertThat(jobsLogSink.get(), Matchers.instanceOf(NoopLogSink.class));
    assertThat(operationsLogSink.get(), Matchers.instanceOf(NoopLogSink.class));
}

Example 5 with JobContextLog

use of io.crate.expression.reference.sys.job.JobContextLog in project crate by crate.

the class RamAccountingQueueSinkTest method testRemoveExpiredLogs.

@Test
public void testRemoveExpiredLogs() {
    StatementClassifier.Classification classification = new StatementClassifier.Classification(Plan.StatementType.SELECT, Collections.singleton("Collect"));
    ConcurrentLinkedQueue<JobContextLog> q = new ConcurrentLinkedQueue<>();
    ScheduledFuture<?> task = TimeBasedQEviction.scheduleTruncate(1_000_000L, 1_000_000L, q, scheduler, TimeValue.timeValueSeconds(1L));
    q.add(new JobContextLog(new JobContext(UUID.fromString("067e6162-3b6f-4ae2-a171-2470b63dff01"), "select 1", 1L, User.CRATE_USER, classification), null, 2000L));
    q.add(new JobContextLog(new JobContext(UUID.fromString("067e6162-3b6f-4ae2-a171-2470b63dff02"), "select 1", 1L, User.CRATE_USER, classification), null, 4000L));
    q.add(new JobContextLog(new JobContext(UUID.fromString("067e6162-3b6f-4ae2-a171-2470b63dff03"), "select 1", 1L, User.CRATE_USER, classification), null, 7000L));
    TimeBasedQEviction.removeExpiredLogs(q, 10_000L, 5_000L);
    assertThat(q.size(), is(1));
    assertThat(q.iterator().next().id(), is(UUID.fromString("067e6162-3b6f-4ae2-a171-2470b63dff03")));
    task.cancel(true);
}