Search in sources :

Example 1 with MigrationFile

use of io.confluent.ksql.tools.migrations.util.MigrationFile in project ksql by confluentinc.

the class ApplyMigrationCommand method apply.

private boolean apply(final MigrationConfig config, final Client ksqlClient, final String migrationsDir, final Clock clock) {
    String previous = MetadataUtil.getLatestMigratedVersion(config, ksqlClient);
    LOGGER.info("Loading migration files");
    final List<MigrationFile> migrations;
    try {
        migrations = loadMigrationsToApply(migrationsDir, previous);
    } catch (MigrationException e) {
        LOGGER.error(e.getMessage());
        return false;
    }
    if (migrations.size() == 0) {
        LOGGER.info("No eligible migrations found.");
    } else {
        LOGGER.info(migrations.size() + " migration file(s) loaded.");
    }
    for (MigrationFile migration : migrations) {
        if (!applyMigration(config, ksqlClient, migration, clock, previous)) {
            return false;
        }
        previous = Integer.toString(migration.getVersion());
    }
    return true;
}
Also used : MigrationException(io.confluent.ksql.tools.migrations.MigrationException) MigrationFile(io.confluent.ksql.tools.migrations.util.MigrationFile)

Example 2 with MigrationFile

use of io.confluent.ksql.tools.migrations.util.MigrationFile in project ksql by confluentinc.

the class MigrationInfoCommand method printAsTable.

private static void printAsTable(final List<MigrationFile> allMigrations, final Map<Integer, Optional<MigrationVersionInfo>> versionInfos) {
    final MigrationVersionInfoFormatter formatter = new MigrationVersionInfoFormatter();
    for (final MigrationFile migration : allMigrations) {
        final MigrationVersionInfo versionInfo = versionInfos.get(migration.getVersion()).orElse(MigrationVersionInfo.pendingMigration(migration.getVersion(), migration.getName()));
        formatter.addVersionInfo(versionInfo);
    }
    LOGGER.info(formatter.getFormatted());
}
Also used : MigrationVersionInfo(io.confluent.ksql.tools.migrations.util.MigrationVersionInfo) MigrationFile(io.confluent.ksql.tools.migrations.util.MigrationFile) MigrationVersionInfoFormatter(io.confluent.ksql.tools.migrations.util.MigrationVersionInfoFormatter)

Example 3 with MigrationFile

use of io.confluent.ksql.tools.migrations.util.MigrationFile in project ksql by confluentinc.

the class ApplyMigrationCommand method executeCommands.

/**
 * If validateOnly is set to true, then this parses each of the commands but only executes
 * DEFINE/UNDEFINE commands (variables are needed for parsing INSERT INTO... VALUES, SET/UNSET
 * and DEFINE commands). If validateOnly is set to false, then each command will execute after
 * parsing.
 */
private void executeCommands(final List<String> commands, final Client ksqlClient, final MigrationConfig config, final String executionStart, final MigrationFile migration, final Clock clock, final String previous, final boolean validateOnly) {
    setUpJavaClientVariables(ksqlClient);
    final Map<String, Object> properties = new HashMap<>();
    for (final String command : commands) {
        try {
            final Map<String, String> variables = ksqlClient.getVariables().entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().toString()));
            executeCommand(CommandParser.transformToSqlCommand(command, variables), ksqlClient, properties, validateOnly);
        } catch (InterruptedException | ExecutionException | MigrationException e) {
            final String action = validateOnly ? "parse" : "execute";
            final String errorMsg = String.format("Failed to %s sql: %s. Error: %s", action, command, e.getMessage());
            updateState(config, ksqlClient, MigrationState.ERROR, executionStart, migration, clock, previous, Optional.of(errorMsg));
            throw new MigrationException(errorMsg);
        }
    }
}
Also used : Option(com.github.rvesse.airline.annotations.Option) MetadataUtil(io.confluent.ksql.tools.migrations.util.MetadataUtil) MigrationState(io.confluent.ksql.tools.migrations.util.MetadataUtil.MigrationState) FieldInfo(io.confluent.ksql.api.client.FieldInfo) MigrationFile(io.confluent.ksql.tools.migrations.util.MigrationFile) MigrationConfig(io.confluent.ksql.tools.migrations.MigrationConfig) IntegerRange(com.github.rvesse.airline.annotations.restrictions.ranges.IntegerRange) SqlCreateConnectorStatement(io.confluent.ksql.tools.migrations.util.CommandParser.SqlCreateConnectorStatement) BiFunction(java.util.function.BiFunction) LoggerFactory(org.slf4j.LoggerFactory) CommandParser(io.confluent.ksql.tools.migrations.util.CommandParser) HashMap(java.util.HashMap) RequireOnlyOne(com.github.rvesse.airline.annotations.restrictions.RequireOnlyOne) SqlCommand(io.confluent.ksql.tools.migrations.util.CommandParser.SqlCommand) MigrationsDirectoryUtil.getAllMigrations(io.confluent.ksql.tools.migrations.util.MigrationsDirectoryUtil.getAllMigrations) SqlInsertValues(io.confluent.ksql.tools.migrations.util.CommandParser.SqlInsertValues) Map(java.util.Map) Once(com.github.rvesse.airline.annotations.restrictions.Once) MigrationsDirectoryUtil.getMigrationsDirFromConfigFile(io.confluent.ksql.tools.migrations.util.MigrationsDirectoryUtil.getMigrationsDirFromConfigFile) SqlDropConnectorStatement(io.confluent.ksql.tools.migrations.util.CommandParser.SqlDropConnectorStatement) Command(com.github.rvesse.airline.annotations.Command) SqlUndefineVariableCommand(io.confluent.ksql.tools.migrations.util.CommandParser.SqlUndefineVariableCommand) SqlDefineVariableCommand(io.confluent.ksql.tools.migrations.util.CommandParser.SqlDefineVariableCommand) VariableParser(io.confluent.ksql.parser.VariableParser) MigrationsDirectoryUtil(io.confluent.ksql.tools.migrations.util.MigrationsDirectoryUtil) Logger(org.slf4j.Logger) Expression(io.confluent.ksql.execution.expression.tree.Expression) MigrationsUtil(io.confluent.ksql.tools.migrations.util.MigrationsUtil) KsqlObject(io.confluent.ksql.api.client.KsqlObject) MigrationsDirectoryUtil.getMigrationForVersion(io.confluent.ksql.tools.migrations.util.MigrationsDirectoryUtil.getMigrationForVersion) RetryUtil(io.confluent.ksql.util.RetryUtil) Collectors(java.util.stream.Collectors) ExecutionException(java.util.concurrent.ExecutionException) List(java.util.List) SqlPropertyCommand(io.confluent.ksql.tools.migrations.util.CommandParser.SqlPropertyCommand) MigrationException(io.confluent.ksql.tools.migrations.MigrationException) SqlStatement(io.confluent.ksql.tools.migrations.util.CommandParser.SqlStatement) KsqlException(io.confluent.ksql.util.KsqlException) Clock(java.time.Clock) Optional(java.util.Optional) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Client(io.confluent.ksql.api.client.Client) CommandParser.preserveCase(io.confluent.ksql.tools.migrations.util.CommandParser.preserveCase) Collections(java.util.Collections) MigrationException(io.confluent.ksql.tools.migrations.MigrationException) HashMap(java.util.HashMap) KsqlObject(io.confluent.ksql.api.client.KsqlObject) ExecutionException(java.util.concurrent.ExecutionException)

Aggregations

MigrationFile (io.confluent.ksql.tools.migrations.util.MigrationFile)3 MigrationException (io.confluent.ksql.tools.migrations.MigrationException)2 Command (com.github.rvesse.airline.annotations.Command)1 Option (com.github.rvesse.airline.annotations.Option)1 Once (com.github.rvesse.airline.annotations.restrictions.Once)1 RequireOnlyOne (com.github.rvesse.airline.annotations.restrictions.RequireOnlyOne)1 IntegerRange (com.github.rvesse.airline.annotations.restrictions.ranges.IntegerRange)1 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Client (io.confluent.ksql.api.client.Client)1 FieldInfo (io.confluent.ksql.api.client.FieldInfo)1 KsqlObject (io.confluent.ksql.api.client.KsqlObject)1 Expression (io.confluent.ksql.execution.expression.tree.Expression)1 VariableParser (io.confluent.ksql.parser.VariableParser)1 MigrationConfig (io.confluent.ksql.tools.migrations.MigrationConfig)1 CommandParser (io.confluent.ksql.tools.migrations.util.CommandParser)1 SqlCommand (io.confluent.ksql.tools.migrations.util.CommandParser.SqlCommand)1 SqlCreateConnectorStatement (io.confluent.ksql.tools.migrations.util.CommandParser.SqlCreateConnectorStatement)1 SqlDefineVariableCommand (io.confluent.ksql.tools.migrations.util.CommandParser.SqlDefineVariableCommand)1 SqlDropConnectorStatement (io.confluent.ksql.tools.migrations.util.CommandParser.SqlDropConnectorStatement)1 SqlInsertValues (io.confluent.ksql.tools.migrations.util.CommandParser.SqlInsertValues)1