Examples with ClusterSettings org.elasticsearch.common.settings.ClusterSettings used on opensource projects

Example 41 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project crate by crate.

the class JobsLogsTest method testRunningOperationsAreNotLostOnSettingsChange.

@Test
public void testRunningOperationsAreNotLostOnSettingsChange() throws Exception {
    ExecutorService executor = Executors.newFixedThreadPool(2);
    Settings settings = Settings.builder().put(JobsLogService.STATS_ENABLED_SETTING.getKey(), true).build();
    JobsLogService jobsLogService = new JobsLogService(settings, clusterService::localNode, clusterSettings, nodeCtx, scheduler, breakerService);
    JobsLogs jobsLogs = jobsLogService.get();
    CountDownLatch latch = new CountDownLatch(2);
    AtomicBoolean doInsertJobs = new AtomicBoolean(true);
    AtomicInteger numJobs = new AtomicInteger();
    int maxQueueSize = JobsLogService.STATS_OPERATIONS_LOG_SIZE_SETTING.getDefault(Settings.EMPTY);
    try {
        executor.submit(() -> {
            while (doInsertJobs.get() && numJobs.get() < maxQueueSize) {
                UUID uuid = UUID.randomUUID();
                jobsLogs.operationStarted(1, uuid, "dummy", () -> -1);
                jobsLogs.operationFinished(1, uuid, null);
                numJobs.incrementAndGet();
            }
            latch.countDown();
        });
        executor.submit(() -> {
            jobsLogService.updateOperationSink(maxQueueSize + 10, JobsLogService.STATS_OPERATIONS_LOG_EXPIRATION_SETTING.getDefault(Settings.EMPTY));
            doInsertJobs.set(false);
            latch.countDown();
        });
        latch.await(10, TimeUnit.SECONDS);
        assertThat(StreamSupport.stream(jobsLogs.operationsLog().spliterator(), false).count(), is((long) numJobs.get()));
    } finally {
        executor.shutdown();
        executor.awaitTermination(2, TimeUnit.SECONDS);
    }
}

Example 42 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project crate by crate.

the class JobsLogsTest method testReEnableStats.

@Test
public void testReEnableStats() {
    clusterService.getClusterSettings().applySettings(Settings.builder().put(JobsLogService.STATS_ENABLED_SETTING.getKey(), false).build());
    Settings settings = Settings.builder().put(JobsLogService.STATS_ENABLED_SETTING.getKey(), false).put(JobsLogService.STATS_JOBS_LOG_SIZE_SETTING.getKey(), 100).put(JobsLogService.STATS_OPERATIONS_LOG_SIZE_SETTING.getKey(), 100).build();
    JobsLogService stats = new JobsLogService(settings, clusterService::localNode, clusterSettings, nodeCtx, scheduler, breakerService);
    LogSink<JobContextLog> jobsLogSink = (LogSink<JobContextLog>) stats.get().jobsLog();
    LogSink<OperationContextLog> operationsLogSink = (LogSink<OperationContextLog>) stats.get().operationsLog();
    assertThat(stats.isEnabled(), is(false));
    assertThat(stats.jobsLogSize, is(100));
    assertThat(jobsLogSink, Matchers.instanceOf(NoopLogSink.class));
    assertThat(stats.operationsLogSize, is(100));
    assertThat(operationsLogSink, Matchers.instanceOf(NoopLogSink.class));
    clusterService.getClusterSettings().applySettings(Settings.builder().put(JobsLogService.STATS_ENABLED_SETTING.getKey(), true).build());
    assertThat(stats.isEnabled(), is(true));
    assertThat(stats.jobsLogSize, is(100));
    assertThat(stats.get().jobsLog(), Matchers.instanceOf(FilteredLogSink.class));
    assertThat(stats.operationsLogSize, is(100));
    assertThat(stats.get().operationsLog(), Matchers.instanceOf(QueueSink.class));
}

Example 43 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project crate by crate.

the class JobsLogsTest method testEntriesCanBeRejectedWithFilter.

@Test
public void testEntriesCanBeRejectedWithFilter() {
    Settings settings = Settings.builder().put(JobsLogService.STATS_JOBS_LOG_FILTER.getKey(), "stmt like 'select%'").build();
    JobsLogService stats = new JobsLogService(settings, clusterService::localNode, clusterSettings, nodeCtx, scheduler, breakerService);
    LogSink<JobContextLog> jobsLogSink = (LogSink<JobContextLog>) stats.get().jobsLog();
    jobsLogSink.add(new JobContextLog(new JobContext(UUID.randomUUID(), "insert into", 10L, User.CRATE_USER, null), null, 20L));
    jobsLogSink.add(new JobContextLog(new JobContext(UUID.randomUUID(), "select * from t1", 10L, User.CRATE_USER, null), null, 20L));
    assertThat(StreamSupport.stream(jobsLogSink.spliterator(), false).count(), is(1L));
}

Example 44 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project crate by crate.

the class JobsLogsTest method testSettingsChanges.

@Test
public void testSettingsChanges() throws Exception {
    Settings settings = Settings.builder().put(JobsLogService.STATS_ENABLED_SETTING.getKey(), true).put(JobsLogService.STATS_JOBS_LOG_SIZE_SETTING.getKey(), 100).put(JobsLogService.STATS_OPERATIONS_LOG_SIZE_SETTING.getKey(), 100).build();
    JobsLogService stats = new JobsLogService(settings, clusterService::localNode, clusterSettings, nodeCtx, scheduler, breakerService);
    Supplier<LogSink<JobContextLog>> jobsLogSink = () -> (LogSink<JobContextLog>) stats.get().jobsLog();
    Supplier<LogSink<OperationContextLog>> operationsLogSink = () -> (LogSink<OperationContextLog>) stats.get().operationsLog();
    // sinks are still of type QueueSink
    assertThat(jobsLogSink.get(), Matchers.instanceOf(FilteredLogSink.class));
    assertThat(operationsLogSink.get(), Matchers.instanceOf(QueueSink.class));
    assertThat(inspectRamAccountingQueue((QueueSink) ((FilteredLogSink) jobsLogSink.get()).delegate), Matchers.instanceOf(BlockingEvictingQueue.class));
    assertThat(inspectRamAccountingQueue((QueueSink) operationsLogSink.get()), Matchers.instanceOf(BlockingEvictingQueue.class));
    clusterSettings.applySettings(Settings.builder().put(JobsLogService.STATS_JOBS_LOG_EXPIRATION_SETTING.getKey(), "10s").put(JobsLogService.STATS_OPERATIONS_LOG_EXPIRATION_SETTING.getKey(), "10s").build());
    assertThat(inspectRamAccountingQueue((QueueSink) ((FilteredLogSink<JobContextLog>) jobsLogSink.get()).delegate), Matchers.instanceOf(ConcurrentLinkedDeque.class));
    assertThat(inspectRamAccountingQueue((QueueSink) operationsLogSink.get()), Matchers.instanceOf(ConcurrentLinkedDeque.class));
    // set all to 0 but don't disable stats
    clusterSettings.applySettings(Settings.builder().put(JobsLogService.STATS_JOBS_LOG_SIZE_SETTING.getKey(), 0).put(JobsLogService.STATS_JOBS_LOG_EXPIRATION_SETTING.getKey(), "0s").put(JobsLogService.STATS_OPERATIONS_LOG_SIZE_SETTING.getKey(), 0).put(JobsLogService.STATS_OPERATIONS_LOG_EXPIRATION_SETTING.getKey(), "0s").build());
    assertThat(jobsLogSink.get(), Matchers.instanceOf(NoopLogSink.class));
    assertThat(operationsLogSink.get(), Matchers.instanceOf(NoopLogSink.class));
    assertThat(stats.isEnabled(), is(true));
    clusterSettings.applySettings(Settings.builder().put(JobsLogService.STATS_JOBS_LOG_SIZE_SETTING.getKey(), 200).put(JobsLogService.STATS_OPERATIONS_LOG_SIZE_SETTING.getKey(), 200).put(JobsLogService.STATS_ENABLED_SETTING.getKey(), true).build());
    assertThat(jobsLogSink.get(), Matchers.instanceOf(FilteredLogSink.class));
    assertThat(inspectRamAccountingQueue((QueueSink) ((FilteredLogSink<JobContextLog>) jobsLogSink.get()).delegate), Matchers.instanceOf(BlockingEvictingQueue.class));
    assertThat(operationsLogSink.get(), Matchers.instanceOf(QueueSink.class));
    assertThat(inspectRamAccountingQueue((QueueSink) operationsLogSink.get()), Matchers.instanceOf(BlockingEvictingQueue.class));
    // disable stats
    clusterSettings.applySettings(Settings.builder().put(JobsLogService.STATS_ENABLED_SETTING.getKey(), false).build());
    assertThat(stats.isEnabled(), is(false));
    assertThat(jobsLogSink.get(), Matchers.instanceOf(NoopLogSink.class));
    assertThat(operationsLogSink.get(), Matchers.instanceOf(NoopLogSink.class));
}

Example 45 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project crate by crate.

the class JobsLogsTest method testRunningJobsAreNotLostOnSettingsChange.

@Test
public void testRunningJobsAreNotLostOnSettingsChange() throws Exception {
    ExecutorService executor = Executors.newFixedThreadPool(2);
    Settings settings = Settings.builder().put(JobsLogService.STATS_ENABLED_SETTING.getKey(), true).build();
    JobsLogService jobsLogService = new JobsLogService(settings, clusterService::localNode, clusterSettings, nodeCtx, scheduler, breakerService);
    JobsLogs jobsLogs = jobsLogService.get();
    Classification classification = new Classification(SELECT, Collections.singleton("Collect"));
    CountDownLatch latch = new CountDownLatch(2);
    AtomicBoolean doInsertJobs = new AtomicBoolean(true);
    AtomicInteger numJobs = new AtomicInteger();
    int maxQueueSize = JobsLogService.STATS_JOBS_LOG_SIZE_SETTING.getDefault(Settings.EMPTY);
    try {
        executor.submit(() -> {
            while (doInsertJobs.get() && numJobs.get() < maxQueueSize) {
                UUID uuid = UUID.randomUUID();
                int i = numJobs.getAndIncrement();
                jobsLogs.logExecutionStart(uuid, "select 1", User.CRATE_USER, classification);
                if (i % 2 == 0) {
                    jobsLogs.logExecutionEnd(uuid, null);
                } else {
                    jobsLogs.logPreExecutionFailure(uuid, "select 1", "failure", User.CRATE_USER);
                }
            }
            latch.countDown();
        });
        executor.submit(() -> {
            jobsLogService.updateJobSink(maxQueueSize + 10, JobsLogService.STATS_JOBS_LOG_EXPIRATION_SETTING.getDefault(Settings.EMPTY));
            doInsertJobs.set(false);
            latch.countDown();
        });
        latch.await(10, TimeUnit.SECONDS);
        assertThat(StreamSupport.stream(jobsLogs.jobsLog().spliterator(), false).count(), is((long) numJobs.get()));
    } finally {
        executor.shutdown();
        executor.awaitTermination(2, TimeUnit.SECONDS);
    }
}