Examples with JobContextLog io.crate.expression.reference.sys.job.JobContextLog used on opensource projects

Example 6 with JobContextLog

use of io.crate.expression.reference.sys.job.JobContextLog in project crate by crate.

the class JobsLogsTest method testExecutionFailure.

@Test
public void testExecutionFailure() {
    JobsLogs jobsLogs = new JobsLogs(() -> true);
    User user = User.of("arthur");
    Queue<JobContextLog> q = new BlockingEvictingQueue<>(1);
    jobsLogs.updateJobsLog(new QueueSink<>(q, () -> {
    }));
    jobsLogs.logPreExecutionFailure(UUID.randomUUID(), "select foo", "stmt error", user);
    List<JobContextLog> jobsLogEntries = StreamSupport.stream(jobsLogs.jobsLog().spliterator(), false).collect(Collectors.toList());
    assertThat(jobsLogEntries.size(), is(1));
    assertThat(jobsLogEntries.get(0).username(), is(user.name()));
    assertThat(jobsLogEntries.get(0).statement(), is("select foo"));
    assertThat(jobsLogEntries.get(0).errorMessage(), is("stmt error"));
    assertThat(jobsLogEntries.get(0).classification(), is(new Classification(UNDEFINED)));
}

Example 7 with JobContextLog

use of io.crate.expression.reference.sys.job.JobContextLog in project crate by crate.

the class JobsLogs method logExecutionEnd.

/**
 * mark a job as finished.
 * <p>
 * If {@link #isEnabled()} is false this method won't do anything.
 */
public void logExecutionEnd(UUID jobId, @Nullable String errorMessage) {
    activeRequests.decrement();
    JobContext jobContext = jobsTable.remove(jobId);
    if (!isEnabled() || jobContext == null) {
        return;
    }
    JobContextLog jobContextLog = new JobContextLog(jobContext, errorMessage);
    recordMetrics(jobContextLog);
    long stamp = jobsLogLock.readLock();
    try {
        jobsLog.add(jobContextLog);
    } finally {
        jobsLogLock.unlockRead(stamp);
    }
}

Example 8 with JobContextLog

use of io.crate.expression.reference.sys.job.JobContextLog in project crate by crate.

the class JobsLogs method logPreExecutionFailure.

/**
 * Create a entry into `sys.jobs_log`
 * This method can be used instead of {@link #logExecutionEnd(UUID, String)} if there was no {@link #logExecutionStart(UUID, String, User, StatementClassifier.Classification)}
 * Call because an error happened during parse, analysis or plan.
 * <p>
 * {@link #logExecutionStart(UUID, String, User, StatementClassifier.Classification)} is only called after a Plan has been created and execution starts.
 */
public void logPreExecutionFailure(UUID jobId, String stmt, String errorMessage, User user) {
    JobContextLog jobContextLog = new JobContextLog(new JobContext(jobId, stmt, System.currentTimeMillis(), user, new StatementClassifier.Classification(UNDEFINED)), errorMessage);
    long stamp = jobsLogLock.readLock();
    try {
        jobsLog.add(jobContextLog);
    } finally {
        jobsLogLock.unlockRead(stamp);
    }
    recordMetrics(jobContextLog);
}

Example 9 with JobContextLog

use of io.crate.expression.reference.sys.job.JobContextLog in project crate by crate.

the class JobsLogsTest method testLogsArentWipedOnSizeChange.

@Test
public void testLogsArentWipedOnSizeChange() {
    Settings settings = Settings.builder().put(JobsLogService.STATS_ENABLED_SETTING.getKey(), true).build();
    JobsLogService stats = new JobsLogService(settings, clusterService::localNode, clusterSettings, nodeCtx, scheduler, breakerService);
    LogSink<JobContextLog> jobsLogSink = (LogSink<JobContextLog>) stats.get().jobsLog();
    LogSink<OperationContextLog> operationsLogSink = (LogSink<OperationContextLog>) stats.get().operationsLog();
    Classification classification = new Classification(SELECT, Collections.singleton("Collect"));
    jobsLogSink.add(new JobContextLog(new JobContext(UUID.randomUUID(), "select 1", 1L, User.CRATE_USER, classification), null));
    clusterSettings.applySettings(Settings.builder().put(JobsLogService.STATS_ENABLED_SETTING.getKey(), true).put(JobsLogService.STATS_JOBS_LOG_SIZE_SETTING.getKey(), 200).build());
    assertThat(StreamSupport.stream(stats.get().jobsLog().spliterator(), false).count(), is(1L));
    operationsLogSink.add(new OperationContextLog(new OperationContext(1, UUID.randomUUID(), "foo", 2L, () -> -1), null));
    operationsLogSink.add(new OperationContextLog(new OperationContext(1, UUID.randomUUID(), "foo", 3L, () -> 1), null));
    clusterSettings.applySettings(Settings.builder().put(JobsLogService.STATS_ENABLED_SETTING.getKey(), true).put(JobsLogService.STATS_OPERATIONS_LOG_SIZE_SETTING.getKey(), 1).build());
    assertThat(StreamSupport.stream(stats.get().operationsLog().spliterator(), false).count(), is(1L));
}

Example 10 with JobContextLog

use of io.crate.expression.reference.sys.job.JobContextLog in project crate by crate.

the class JobLogIntegrationTest method testJobLogWithEnabledAndDisabledStats.

@Test
// SET extra_float_digits = 3 gets added to the jobs_log
@UseJdbc(0)
public void testJobLogWithEnabledAndDisabledStats() throws Exception {
    String setStmt = "set global transient stats.jobs_log_size=1";
    execute(setStmt);
    // We record the statements in the log **after** we notify the result receivers (see {@link JobsLogsUpdateListener usage).
    // So it might happen that the "set global ..." statement execution is returned to this test but the recording
    // in the log is done AFTER the execution of the below "select name from sys.cluster" statement (because async
    // programming is evil like that). And then this test will fail and people will spend days and days to figure
    // out what's going on.
    // So let's just wait for the "set global ... " statement to be recorded here and then move on with our test.
    assertBusy(() -> {
        boolean setStmtFound = false;
        for (JobsLogService jobsLogService : internalCluster().getDataNodeInstances(JobsLogService.class)) {
            // each node must have received the new jobs_log_size setting change instruction
            assertThat(jobsLogService.jobsLogSize(), is(1));
            JobsLogs jobsLogs = jobsLogService.get();
            Iterator<JobContextLog> iterator = jobsLogs.jobsLog().iterator();
            if (iterator.hasNext()) {
                if (iterator.next().statement().equalsIgnoreCase(setStmt)) {
                    setStmtFound = true;
                }
            }
        }
        // at least one node must have the set statement logged
        assertThat(setStmtFound, is(true));
    });
    // only the latest queries are found in the log.
    for (SQLOperations sqlOperations : internalCluster().getDataNodeInstances(SQLOperations.class)) {
        Session session = sqlOperations.newSystemSession();
        execute("select name from sys.cluster", null, session);
    }
    assertJobLogOnNodesHaveOnlyStatement("select name from sys.cluster");
    for (SQLOperations sqlOperations : internalCluster().getDataNodeInstances(SQLOperations.class)) {
        Session session = sqlOperations.newSystemSession();
        execute("select id from sys.cluster", null, session);
    }
    assertJobLogOnNodesHaveOnlyStatement("select id from sys.cluster");
    execute("set global transient stats.enabled = false");
    for (JobsLogService jobsLogService : internalCluster().getDataNodeInstances(JobsLogService.class)) {
        assertBusy(() -> assertThat(jobsLogService.isEnabled(), is(false)));
    }
    execute("select * from sys.jobs_log");
    assertThat(response.rowCount(), is(0L));
}