Search in sources :

Example 26 with CommandId

use of io.confluent.ksql.rest.entity.CommandId in project ksql by confluentinc.

the class StatusResourceTest method testGetAllStatuses.

@Test
public void testGetAllStatuses() {
    final StatusResource testResource = getTestStatusResource();
    final Object statusesEntity = testResource.getAllStatuses().getEntity();
    assertThat(statusesEntity, instanceOf(CommandStatuses.class));
    final CommandStatuses testCommandStatuses = (CommandStatuses) statusesEntity;
    final Map<CommandId, CommandStatus.Status> expectedCommandStatuses = CommandStatuses.fromFullStatuses(mockCommandStatuses);
    assertEquals(expectedCommandStatuses, testCommandStatuses);
}
Also used : CommandStatus(io.confluent.ksql.rest.entity.CommandStatus) CommandStatuses(io.confluent.ksql.rest.entity.CommandStatuses) EasyMock.anyObject(org.easymock.EasyMock.anyObject) CommandId(io.confluent.ksql.rest.entity.CommandId) Test(org.junit.Test)

Example 27 with CommandId

use of io.confluent.ksql.rest.entity.CommandId in project ksql by confluentinc.

the class StatusResourceTest method getTestStatusResource.

private StatusResource getTestStatusResource() {
    final InteractiveStatementExecutor mockStatementExecutor = mock(InteractiveStatementExecutor.class);
    expect(mockStatementExecutor.getStatuses()).andReturn(mockCommandStatuses);
    for (final Map.Entry<CommandId, CommandStatus> commandEntry : mockCommandStatuses.entrySet()) {
        expect(mockStatementExecutor.getStatus(commandEntry.getKey())).andReturn(Optional.of(commandEntry.getValue()));
    }
    expect(mockStatementExecutor.getStatus(anyObject(CommandId.class))).andReturn(Optional.empty());
    replay(mockStatementExecutor);
    return new StatusResource(mockStatementExecutor);
}
Also used : InteractiveStatementExecutor(io.confluent.ksql.rest.server.computation.InteractiveStatementExecutor) CommandStatus(io.confluent.ksql.rest.entity.CommandStatus) CommandId(io.confluent.ksql.rest.entity.CommandId) HashMap(java.util.HashMap) Map(java.util.Map)

Example 28 with CommandId

use of io.confluent.ksql.rest.entity.CommandId in project ksql by confluentinc.

the class DistributingExecutor method execute.

/**
 * The transactional protocol for sending a command to the command topic is to
 * initTransaction(), beginTransaction(), wait for commandRunner to finish processing all previous
 * commands that were present at the start of the transaction, validate the current command,
 * enqueue the command in the command topic, and commit the transaction.
 * Only successfully committed commands can be read by the command topic consumer.
 * If any exceptions are thrown during this protocol, the transaction is aborted.
 * If a new transactional producer is initialized while the current transaction is incomplete,
 * the old producer will be fenced off and unable to continue with its transaction.
 */
// CHECKSTYLE_RULES.OFF: NPathComplexity
public StatementExecutorResponse execute(final ConfiguredStatement<? extends Statement> statement, final KsqlExecutionContext executionContext, final KsqlSecurityContext securityContext) {
    final String commandRunnerWarningString = commandRunnerWarning.get();
    if (!commandRunnerWarningString.equals("")) {
        throw new KsqlServerException("Failed to handle Ksql Statement." + System.lineSeparator() + commandRunnerWarningString);
    }
    final ConfiguredStatement<?> injected = injectorFactory.apply(executionContext, securityContext.getServiceContext()).inject(statement);
    if (injected.getStatement() instanceof InsertInto) {
        validateInsertIntoQueries(executionContext.getMetaStore(), (InsertInto) injected.getStatement());
    }
    final Optional<StatementExecutorResponse> response = checkIfNotExistsResponse(executionContext, statement);
    if (response.isPresent()) {
        return response.get();
    }
    checkAuthorization(injected, securityContext, executionContext);
    final Producer<CommandId, Command> transactionalProducer = commandQueue.createTransactionalProducer();
    try {
        transactionalProducer.initTransactions();
    } catch (final TimeoutException e) {
        throw new KsqlServerException(errorHandler.transactionInitTimeoutErrorMessage(e), e);
    } catch (final Exception e) {
        throw new KsqlServerException(String.format("Could not write the statement '%s' into the command topic: " + e.getMessage(), statement.getStatementText()), e);
    }
    if (!rateLimiter.tryAcquire(1, TimeUnit.SECONDS)) {
        throw new KsqlRestException(Errors.tooManyRequests("DDL/DML rate is crossing the configured rate limit of statements/second"));
    }
    CommandId commandId = null;
    try {
        transactionalProducer.beginTransaction();
        commandQueue.waitForCommandConsumer();
        commandId = commandIdAssigner.getCommandId(statement.getStatement());
        final Command command = validatedCommandFactory.create(injected, executionContext.createSandbox(executionContext.getServiceContext()));
        final QueuedCommandStatus queuedCommandStatus = commandQueue.enqueueCommand(commandId, command, transactionalProducer);
        transactionalProducer.commitTransaction();
        final CommandStatus commandStatus = queuedCommandStatus.tryWaitForFinalStatus(distributedCmdResponseTimeout);
        return StatementExecutorResponse.handled(Optional.of(new CommandStatusEntity(injected.getStatementText(), queuedCommandStatus.getCommandId(), commandStatus, queuedCommandStatus.getCommandSequenceNumber(), getDeprecatedWarnings(executionContext.getMetaStore(), injected))));
    } catch (final ProducerFencedException | OutOfOrderSequenceException | AuthorizationException e) {
        // This catch doesn't abortTransaction() since doing that would throw another exception.
        if (commandId != null) {
            commandQueue.abortCommand(commandId);
        }
        throw new KsqlServerException(String.format("Could not write the statement '%s' into the command topic.", statement.getStatementText()), e);
    } catch (final Exception e) {
        transactionalProducer.abortTransaction();
        if (commandId != null) {
            commandQueue.abortCommand(commandId);
        }
        throw new KsqlServerException(String.format("Could not write the statement '%s' into the command topic.", statement.getStatementText()), e);
    } finally {
        transactionalProducer.close();
    }
}
Also used : AuthorizationException(org.apache.kafka.common.errors.AuthorizationException) KsqlServerException(io.confluent.ksql.util.KsqlServerException) StatementExecutorResponse(io.confluent.ksql.rest.server.execution.StatementExecutorResponse) TimeoutException(org.apache.kafka.common.errors.TimeoutException) AuthorizationException(org.apache.kafka.common.errors.AuthorizationException) OutOfOrderSequenceException(org.apache.kafka.common.errors.OutOfOrderSequenceException) KsqlServerException(io.confluent.ksql.util.KsqlServerException) ProducerFencedException(org.apache.kafka.common.errors.ProducerFencedException) KsqlRestException(io.confluent.ksql.rest.server.resources.KsqlRestException) KsqlException(io.confluent.ksql.util.KsqlException) CommandStatusEntity(io.confluent.ksql.rest.entity.CommandStatusEntity) ProducerFencedException(org.apache.kafka.common.errors.ProducerFencedException) OutOfOrderSequenceException(org.apache.kafka.common.errors.OutOfOrderSequenceException) KsqlRestException(io.confluent.ksql.rest.server.resources.KsqlRestException) InsertInto(io.confluent.ksql.parser.tree.InsertInto) CommandStatus(io.confluent.ksql.rest.entity.CommandStatus) CommandId(io.confluent.ksql.rest.entity.CommandId) TimeoutException(org.apache.kafka.common.errors.TimeoutException)

Example 29 with CommandId

use of io.confluent.ksql.rest.entity.CommandId in project ksql by confluentinc.

the class CommandStore method enqueueCommand.

@Override
public QueuedCommandStatus enqueueCommand(final CommandId commandId, final Command command, final Producer<CommandId, Command> transactionalProducer) {
    final CommandStatusFuture statusFuture = commandStatusMap.compute(commandId, (k, v) -> {
        if (v == null) {
            return new CommandStatusFuture(commandId);
        }
        // a caller from receiving a future for a different statement.
        throw new IllegalStateException(String.format("Another command with the same id (%s) is being executed.", commandId));
    });
    try {
        final ProducerRecord<CommandId, Command> producerRecord = new ProducerRecord<>(commandTopicName, COMMAND_TOPIC_PARTITION, commandId, command);
        final RecordMetadata recordMetadata = transactionalProducer.send(producerRecord).get();
        return new QueuedCommandStatus(recordMetadata.offset(), statusFuture);
    } catch (final Exception e) {
        commandStatusMap.remove(commandId);
        throw new KsqlException(String.format("Could not write the statement '%s' into the " + "command topic" + ".", command.getStatement()), e);
    }
}
Also used : RecordMetadata(org.apache.kafka.clients.producer.RecordMetadata) ProducerRecord(org.apache.kafka.clients.producer.ProducerRecord) CommandId(io.confluent.ksql.rest.entity.CommandId) KsqlException(io.confluent.ksql.util.KsqlException) TimeoutException(java.util.concurrent.TimeoutException) ExecutionException(java.util.concurrent.ExecutionException) KsqlServerException(io.confluent.ksql.util.KsqlServerException) KsqlException(io.confluent.ksql.util.KsqlException)

Aggregations

CommandId (io.confluent.ksql.rest.entity.CommandId)29 Test (org.junit.Test)20 CommandStatus (io.confluent.ksql.rest.entity.CommandStatus)13 CommandStatusEntity (io.confluent.ksql.rest.entity.CommandStatusEntity)7 QueryId (io.confluent.ksql.query.QueryId)6 CoreMatchers.containsString (org.hamcrest.CoreMatchers.containsString)6 BaseApiTest (io.confluent.ksql.api.BaseApiTest)5 KsqlException (io.confluent.ksql.util.KsqlException)5 Command (io.confluent.ksql.rest.server.computation.Command)4 JsonObject (io.vertx.core.json.JsonObject)4 Matchers.containsString (org.hamcrest.Matchers.containsString)4 IntegrationTest (io.confluent.common.utils.IntegrationTest)3 KsqlServerException (io.confluent.ksql.util.KsqlServerException)3 PersistentQueryMetadata (io.confluent.ksql.util.PersistentQueryMetadata)3 IOException (java.io.IOException)3 HashMap (java.util.HashMap)3 Map (java.util.Map)3 ExecutionException (java.util.concurrent.ExecutionException)3 TimeoutException (java.util.concurrent.TimeoutException)3 KsqlPlan (io.confluent.ksql.engine.KsqlPlan)2