Examples with SchemaUpdateResult io.datarouter.storage.config.schema.SchemaUpdateResult used on opensource projects

Example 1 with SchemaUpdateResult

use of io.datarouter.storage.config.schema.SchemaUpdateResult in project datarouter by hotpads.

the class HBaseSchemaUpdateService method generateSchemaUpdate.

private Optional<SchemaUpdateResult> generateSchemaUpdate(ClientId clientId, Supplier<List<String>> existingTableNames, PhysicalNode<?, ?, ?> node) throws IOException {
    PhysicalDatabeanFieldInfo<?, ?, ?> fieldInfo = node.getFieldInfo();
    TableName tableName = TableName.valueOf(fieldInfo.getTableName());
    if (!existingTableNames.get().contains(tableName.getNameAsString())) {
        createTable(clientId, node);
        return Optional.empty();
    }
    Admin admin = hBaseConnectionHolder.getConnection(clientId).getAdmin();
    admin.getAlterStatus(tableName);
    HTableDescriptor desc = admin.getTableDescriptor(tableName);
    int requestedTtlSeconds = fieldInfo.getSampleFielder().getOption(TtlFielderConfig.KEY).map(TtlFielderConfig::getTtl).map(Duration::getSeconds).map(Math::toIntExact).orElse(HConstants.FOREVER);
    List<String> ddls = new ArrayList<>();
    for (HColumnDescriptor column : desc.getColumnFamilies()) {
        if (requestedTtlSeconds != column.getTimeToLive()) {
            String ddl = "alter '" + tableName + "', NAME => '" + column.getNameAsString() + "', TTL => " + requestedTtlSeconds;
            if (schemaUpdateOptions.getModifyTtl(false)) {
                logger.warn(SchemaUpdateTool.generateFullWidthMessage("Executing SchemaUpdate"));
                logger.warn(ddl);
                column.setTimeToLive(requestedTtlSeconds);
                admin.modifyColumn(tableName, column);
            } else if (schemaUpdateOptions.getModifyTtl(true)) {
                SchemaUpdateTool.printSchemaUpdate(logger, ddl);
                ddls.add(ddl);
            }
        }
        if (MAX_VERSIONS != column.getMaxVersions()) {
            String ddl = "alter '" + tableName + "', NAME => '" + column.getNameAsString() + "', VERSIONS => " + MAX_VERSIONS;
            if (schemaUpdateOptions.getModifyMaxVersions(false)) {
                logger.warn(SchemaUpdateTool.generateFullWidthMessage("Executing SchemaUpdate"));
                logger.warn(ddl);
                column.setMaxVersions(MAX_VERSIONS);
                admin.modifyColumn(tableName, column);
            } else if (schemaUpdateOptions.getModifyMaxVersions(true)) {
                SchemaUpdateTool.printSchemaUpdate(logger, ddl);
                ddls.add(ddl);
            }
        }
    }
    if (ddls.isEmpty()) {
        return Optional.empty();
    }
    return Optional.of(new SchemaUpdateResult(String.join("\n", ddls), null, clientId));
}

Example 2 with SchemaUpdateResult

use of io.datarouter.storage.config.schema.SchemaUpdateResult in project datarouter by hotpads.

the class MysqlSingleTableSchemaUpdateService method performSchemaUpdate.

public Optional<SchemaUpdateResult> performSchemaUpdate(ClientId clientId, Supplier<List<String>> existingTableNames, PhysicalNode<?, ?, ?> physicalNode) {
    MysqlConnectionPool connectionPool = mysqlConnectionPoolHolder.getConnectionPool(clientId);
    if (schemaUpdateOptions.getIgnoreClients().contains(clientId.getName()) || !clientId.getWritable()) {
        return Optional.empty();
    }
    String tableName = physicalNode.getFieldInfo().getTableName();
    String currentTableAbsoluteName = clientId.getName() + "." + tableName;
    if (schemaUpdateOptions.getIgnoreTables().contains(currentTableAbsoluteName)) {
        return Optional.empty();
    }
    String schemaName = connectionPool.getSchemaName();
    DatabeanFieldInfo<?, ?, ?> fieldInfo = physicalNode.getFieldInfo();
    List<Field<?>> primaryKeyFields = fieldInfo.getPrimaryKeyFields();
    List<Field<?>> nonKeyFields = fieldInfo.getNonKeyFields();
    Map<String, List<Field<?>>> indexes = Collections.emptyMap();
    Map<String, List<Field<?>>> uniqueIndexes = fieldInfo.getUniqueIndexes();
    MysqlTableOptions mysqlTableOptions = MysqlTableOptions.make(fieldInfo.getSampleFielder());
    MysqlCollation collation = mysqlTableOptions.getCollation();
    MysqlCharacterSet characterSet = mysqlTableOptions.getCharacterSet();
    MysqlRowFormat rowFormat = mysqlTableOptions.getRowFormat();
    if (physicalNode instanceof IndexedStorage) {
        IndexedStorage<?, ?> indexedStorage = (IndexedStorage<?, ?>) physicalNode;
        indexes = indexedStorage.getManagedNodes().stream().collect(Collectors.toMap(ManagedNode::getName, managedNode -> managedNode.getIndexEntryFieldInfo().getFields()));
    }
    SqlTable requested = fieldSqlTableGenerator.generate(tableName, primaryKeyFields, nonKeyFields, collation, characterSet, rowFormat, indexes, uniqueIndexes);
    boolean exists = existingTableNames.get().contains(tableName);
    if (!exists) {
        String createDdl = sqlCreateTableGenerator.generateDdl(requested, schemaName);
        if (schemaUpdateOptions.getCreateTables(false)) {
            logger.info(SchemaUpdateTool.generateFullWidthMessage("Creating the table " + tableName));
            logger.info(createDdl);
            try {
                MysqlTool.execute(connectionPool, createDdl);
                logger.info(SchemaUpdateTool.generateFullWidthMessage("Created " + tableName));
            } catch (RuntimeException e) {
                Throwable cause = e.getCause();
                if (!(cause instanceof SQLSyntaxErrorException)) {
                    throw e;
                }
                int errorCode = ((SQLSyntaxErrorException) e.getCause()).getErrorCode();
                if (errorCode != MysqlErrorNumbers.ER_TABLE_EXISTS_ERROR) {
                    throw e;
                }
                logger.warn(SchemaUpdateTool.generateFullWidthMessage("Did not create " + tableName + " because it already exists"));
            }
            return Optional.empty();
        }
        SchemaUpdateTool.printSchemaUpdate(logger, createDdl);
        return Optional.of(new SchemaUpdateResult(createDdl, tableName + " creation is required", clientId));
    }
    SqlTable executeCurrent = ConnectionSqlTableGenerator.generate(connectionPool, tableName, schemaName);
    SqlAlterTableGenerator executeAlterTableGenerator = sqlAlterTableGeneratorFactory.new SqlAlterTableGenerator(executeCurrent, requested, mysqlOptions.hostname(clientId), schemaName);
    // execute the alter table
    Ddl ddl = executeAlterTableGenerator.generateDdl();
    if (ddl.executeStatement.isPresent()) {
        PhaseTimer alterTableTimer = new PhaseTimer();
        logger.info(SchemaUpdateTool.generateFullWidthMessage("Executing " + getClass().getSimpleName() + " SchemaUpdate"));
        logger.info(ddl.executeStatement.get());
        MysqlTool.execute(connectionPool, ddl.executeStatement.get());
        alterTableTimer.add("Completed SchemaUpdate for " + tableName);
        logger.info(SchemaUpdateTool.generateFullWidthMessage(alterTableTimer.getElapsedString()));
    }
    if (ddl.printStatement.isEmpty()) {
        return Optional.empty();
    }
    SchemaUpdateTool.printSchemaUpdate(logger, ddl.printStatement.get());
    String errorMessage = null;
    if (ddl.preventStartUp) {
        errorMessage = "an alter on " + tableName + " is required";
    }
    return Optional.of(new SchemaUpdateResult(ddl.printStatement.get(), errorMessage, clientId));
}

Example 3 with SchemaUpdateResult

use of io.datarouter.storage.config.schema.SchemaUpdateResult in project datarouter by hotpads.

the class SpannerSingleTableSchemaUpdateService method performSchemaUpdate.

public Optional<SchemaUpdateResult> performSchemaUpdate(ClientId clientId, Supplier<List<String>> existingTableNames, PhysicalNode<?, ?, ?> physicalNode) {
    String tableName = physicalNode.getFieldInfo().getTableName();
    List<Field<?>> primaryKeyFields = physicalNode.getFieldInfo().getSamplePrimaryKey().getFields();
    List<? extends SpannerBaseFieldCodec<?, ?>> primaryKeyCodecs = fieldCodecRegistry.createCodecs(primaryKeyFields);
    for (SpannerBaseFieldCodec<?, ?> codec : primaryKeyCodecs) {
        if (codec.getSpannerColumnType().isArray()) {
            throw new RuntimeException("Invalid field type used for primary key: " + codec.getField().getKey().getName());
        }
    }
    List<SpannerIndex> indexes = new ArrayList<>();
    List<SpannerIndex> uniqueIndexes = Scanner.of(physicalNode.getFieldInfo().getUniqueIndexes().entrySet()).map(entry -> new SpannerIndex(tableName, entry.getKey(), entry.getValue(), Collections.emptyList(), true)).list();
    var statements = new SpannerUpdateStatements();
    String entityTableName = null;
    if (physicalNode instanceof IndexedStorage) {
        IndexedStorage<?, ?> indexedStorage = (IndexedStorage<?, ?>) physicalNode;
        indexes = Scanner.of(indexedStorage.getManagedNodes()).map(node -> new SpannerIndex(tableName, node.getName(), node.getIndexEntryFieldInfo().getPrimaryKeyFields(), node.getIndexEntryFieldInfo().getFields(), false)).list();
    }
    List<SpannerColumn> primaryKeyColumns = Scanner.of(primaryKeyCodecs).map(codec -> codec.getSpannerColumn(false)).list();
    List<SpannerColumn> nonKeyColumns = Scanner.of(fieldCodecRegistry.createCodecs(physicalNode.getFieldInfo().getNonKeyFields())).map(codec -> codec.getSpannerColumn(true)).list();
    if (!existingTableNames.get().contains(tableName)) {
        statements.updateFunction(tableOperationsGenerator.createTable(tableName, primaryKeyColumns, nonKeyColumns, entityTableName), updateOptions::getCreateTables, true);
        Scanner.of(indexes, uniqueIndexes).concat(Scanner::of).map(index -> createIndex(index, primaryKeyColumns)).forEach(statement -> statements.updateFunction(statement, updateOptions::getCreateTables, true));
    } else {
        DatabaseClient databaseClient = clientsHolder.getDatabaseClient(clientId);
        List<SpannerColumn> allColumns = Scanner.of(primaryKeyColumns, nonKeyColumns).concat(Scanner::of).list();
        ResultSet columnRs = databaseClient.singleUse().executeQuery(Statement.of(tableOperationsGenerator.getTableSchema(tableName)));
        ResultSet primaryKeyRs = databaseClient.singleUse().executeQuery(Statement.of(tableOperationsGenerator.getTableIndexColumnsSchema(tableName, "PRIMARY_KEY")));
        tableAlterSchemaService.generateUpdateStatementColumns(tableName, allColumns, primaryKeyColumns, columnRs, primaryKeyRs, statements);
        ResultSet indexesRs = databaseClient.singleUse().executeQuery(Statement.of(tableOperationsGenerator.getTableIndexSchema(tableName)));
        Set<String> currentIndexes = tableAlterSchemaService.getIndexes(indexesRs);
        Scanner.of(indexes, uniqueIndexes).concat(Scanner::of).forEach(index -> {
            Statement tableIndexColumnsSchema = Statement.of(tableOperationsGenerator.getTableIndexColumnsSchema(tableName, index.getIndexName()));
            ResultSet indexRs = databaseClient.singleUse().executeQuery(tableIndexColumnsSchema);
            if (!tableAlterSchemaService.indexEqual(index, indexRs)) {
                if (currentIndexes.contains(index.getIndexName())) {
                    statements.updateFunction(tableOperationsGenerator.dropIndex(index.getIndexName()), updateOptions::getDropIndexes, false);
                }
                statements.updateFunction(createIndex(index, primaryKeyColumns), updateOptions::getAddIndexes, true);
            }
            currentIndexes.remove(index.getIndexName());
        });
        currentIndexes.forEach(name -> statements.updateFunction(tableOperationsGenerator.dropIndex(name), updateOptions::getDropIndexes, false));
    }
    String errorMessage = null;
    if (!statements.getExecuteStatements().isEmpty()) {
        logger.info(SchemaUpdateTool.generateFullWidthMessage("Executing Spanner " + getClass().getSimpleName() + " SchemaUpdate"));
        logger.info(String.join("\n\n", statements.getExecuteStatements()));
        Database database = clientsHolder.getDatabase(clientId);
        OperationFuture<Void, UpdateDatabaseDdlMetadata> future = database.updateDdl(statements.getExecuteStatements(), null);
        errorMessage = FutureTool.get(future.getPollingFuture().getAttemptResult()).getErrorMessage();
        if (StringTool.notNullNorEmptyNorWhitespace(errorMessage)) {
            logger.error(errorMessage);
        }
    }
    if (statements.getPreventStartUp()) {
        errorMessage = "an alter on Spanner table " + tableName + " is required";
    }
    if (statements.getPrintStatements().isEmpty()) {
        return Optional.empty();
    }
    String printStatement = statements.getPrintStatements().stream().map(statement -> statement + ";").collect(Collectors.joining("\n"));
    SchemaUpdateTool.printSchemaUpdate(logger, printStatement);
    return Optional.of(new SchemaUpdateResult(printStatement, errorMessage, clientId));
}