Examples with JobsLogs io.crate.execution.engine.collect.stats.JobsLogs used on opensource projects

Example 11 with JobsLogs

use of io.crate.execution.engine.collect.stats.JobsLogs in project crate by crate.

the class Session method quickExec.

/**
 * Execute a query in one step, avoiding the parse/bind/execute/sync procedure.
 * Opposed to using parse/bind/execute/sync this method is thread-safe.
 *
 * @param parse A function to parse the statement; This can be used to cache the parsed statement.
 *              Use {@link #quickExec(String, ResultReceiver, Row)} to use the regular parser
 */
public void quickExec(String statement, Function<String, Statement> parse, ResultReceiver<?> resultReceiver, Row params) {
    CoordinatorTxnCtx txnCtx = new CoordinatorTxnCtx(sessionContext);
    Statement parsedStmt = parse.apply(statement);
    AnalyzedStatement analyzedStatement = analyzer.analyze(parsedStmt, sessionContext, ParamTypeHints.EMPTY);
    RoutingProvider routingProvider = new RoutingProvider(Randomness.get().nextInt(), planner.getAwarenessAttributes());
    UUID jobId = UUIDs.dirtyUUID();
    ClusterState clusterState = planner.currentClusterState();
    PlannerContext plannerContext = new PlannerContext(clusterState, routingProvider, jobId, txnCtx, nodeCtx, 0, params);
    Plan plan;
    try {
        plan = planner.plan(analyzedStatement, plannerContext);
    } catch (Throwable t) {
        jobsLogs.logPreExecutionFailure(jobId, statement, SQLExceptions.messageOf(t), sessionContext.sessionUser());
        throw t;
    }
    StatementClassifier.Classification classification = StatementClassifier.classify(plan);
    jobsLogs.logExecutionStart(jobId, statement, sessionContext.sessionUser(), classification);
    JobsLogsUpdateListener jobsLogsUpdateListener = new JobsLogsUpdateListener(jobId, jobsLogs);
    if (!analyzedStatement.isWriteOperation()) {
        resultReceiver = new RetryOnFailureResultReceiver(executor.clusterService(), clusterState, // clusterState at the time of the index check is used
        indexName -> clusterState.metadata().hasIndex(indexName), resultReceiver, jobId, (newJobId, retryResultReceiver) -> retryQuery(newJobId, analyzedStatement, routingProvider, new RowConsumerToResultReceiver(retryResultReceiver, 0, jobsLogsUpdateListener), params, txnCtx, nodeCtx));
    }
    RowConsumerToResultReceiver consumer = new RowConsumerToResultReceiver(resultReceiver, 0, jobsLogsUpdateListener);
    plan.execute(executor, plannerContext, consumer, params, SubQueryResults.EMPTY);
}

Example 12 with JobsLogs

use of io.crate.execution.engine.collect.stats.JobsLogs in project crate by crate.

the class Session method bulkExec.

private CompletableFuture<?> bulkExec(Statement statement, List<DeferredExecution> toExec) {
    assert toExec.size() >= 1 : "Must have at least 1 deferred execution for bulk exec";
    var jobId = UUIDs.dirtyUUID();
    var routingProvider = new RoutingProvider(Randomness.get().nextInt(), planner.getAwarenessAttributes());
    var clusterState = executor.clusterService().state();
    var txnCtx = new CoordinatorTxnCtx(sessionContext);
    var plannerContext = new PlannerContext(clusterState, routingProvider, jobId, txnCtx, nodeCtx, 0, null);
    PreparedStmt firstPreparedStatement = toExec.get(0).portal().preparedStmt();
    AnalyzedStatement analyzedStatement = firstPreparedStatement.analyzedStatement();
    Plan plan;
    try {
        plan = planner.plan(analyzedStatement, plannerContext);
    } catch (Throwable t) {
        jobsLogs.logPreExecutionFailure(jobId, firstPreparedStatement.rawStatement(), SQLExceptions.messageOf(t), sessionContext.sessionUser());
        throw t;
    }
    jobsLogs.logExecutionStart(jobId, firstPreparedStatement.rawStatement(), sessionContext.sessionUser(), StatementClassifier.classify(plan));
    var bulkArgs = Lists2.map(toExec, x -> (Row) new RowN(x.portal().params().toArray()));
    List<CompletableFuture<Long>> rowCounts = plan.executeBulk(executor, plannerContext, bulkArgs, SubQueryResults.EMPTY);
    CompletableFuture<Void> allRowCounts = CompletableFuture.allOf(rowCounts.toArray(new CompletableFuture[0]));
    List<CompletableFuture<?>> resultReceiverFutures = Lists2.map(toExec, x -> x.resultReceiver().completionFuture());
    CompletableFuture<Void> allResultReceivers = CompletableFuture.allOf(resultReceiverFutures.toArray(new CompletableFuture[0]));
    return allRowCounts.exceptionally(// swallow exception - failures are set per item in emitResults
    t -> null).thenAccept(ignored -> emitRowCountsToResultReceivers(jobId, jobsLogs, toExec, rowCounts)).runAfterBoth(allResultReceivers, () -> {
    });
}

Example 13 with JobsLogs

use of io.crate.execution.engine.collect.stats.JobsLogs in project crate by crate.

the class JobLogIntegrationTest method testJobLogWithEnabledAndDisabledStats.

@Test
// SET extra_float_digits = 3 gets added to the jobs_log
@UseJdbc(0)
public void testJobLogWithEnabledAndDisabledStats() throws Exception {
    String setStmt = "set global transient stats.jobs_log_size=1";
    execute(setStmt);
    // We record the statements in the log **after** we notify the result receivers (see {@link JobsLogsUpdateListener usage).
    // So it might happen that the "set global ..." statement execution is returned to this test but the recording
    // in the log is done AFTER the execution of the below "select name from sys.cluster" statement (because async
    // programming is evil like that). And then this test will fail and people will spend days and days to figure
    // out what's going on.
    // So let's just wait for the "set global ... " statement to be recorded here and then move on with our test.
    assertBusy(() -> {
        boolean setStmtFound = false;
        for (JobsLogService jobsLogService : internalCluster().getDataNodeInstances(JobsLogService.class)) {
            // each node must have received the new jobs_log_size setting change instruction
            assertThat(jobsLogService.jobsLogSize(), is(1));
            JobsLogs jobsLogs = jobsLogService.get();
            Iterator<JobContextLog> iterator = jobsLogs.jobsLog().iterator();
            if (iterator.hasNext()) {
                if (iterator.next().statement().equalsIgnoreCase(setStmt)) {
                    setStmtFound = true;
                }
            }
        }
        // at least one node must have the set statement logged
        assertThat(setStmtFound, is(true));
    });
    // only the latest queries are found in the log.
    for (SQLOperations sqlOperations : internalCluster().getDataNodeInstances(SQLOperations.class)) {
        Session session = sqlOperations.newSystemSession();
        execute("select name from sys.cluster", null, session);
    }
    assertJobLogOnNodesHaveOnlyStatement("select name from sys.cluster");
    for (SQLOperations sqlOperations : internalCluster().getDataNodeInstances(SQLOperations.class)) {
        Session session = sqlOperations.newSystemSession();
        execute("select id from sys.cluster", null, session);
    }
    assertJobLogOnNodesHaveOnlyStatement("select id from sys.cluster");
    execute("set global transient stats.enabled = false");
    for (JobsLogService jobsLogService : internalCluster().getDataNodeInstances(JobsLogService.class)) {
        assertBusy(() -> assertThat(jobsLogService.isEnabled(), is(false)));
    }
    execute("select * from sys.jobs_log");
    assertThat(response.rowCount(), is(0L));
}

Example 14 with JobsLogs

use of io.crate.execution.engine.collect.stats.JobsLogs in project crate by crate.

the class JobLogIntegrationTest method assertJobLogOnNodesHaveOnlyStatement.

private void assertJobLogOnNodesHaveOnlyStatement(String statement) throws Exception {
    for (JobsLogService jobsLogService : internalCluster().getDataNodeInstances(JobsLogService.class)) {
        assertBusy(() -> {
            assertThat(jobsLogService.jobsLogSize(), is(1));
            JobsLogs jobsLogs = jobsLogService.get();
            Iterator<JobContextLog> iterator = jobsLogs.jobsLog().iterator();
            if (iterator.hasNext()) {
                assertThat(iterator.next().statement(), is(statement));
            }
        });
    }
}