Examples with ProgressListener herddb.backup.ProgressListener used on opensource projects

Example 1 with ProgressListener

use of herddb.backup.ProgressListener in project herddb by diennea.

the class HerdDBCLI method performRestore.

private static void performRestore(String file, String leader, String newschema, Options options, final Statement statement, final Connection connection) throws SQLException, Exception {
    if (file.isEmpty()) {
        println("Please provide --file option");
        failAndPrintHelp(options);
        return;
    }
    Path inputfile = Paths.get(file).toAbsolutePath();
    if (leader.isEmpty() || newschema.isEmpty()) {
        println("options 'newleader' and 'newschema' are required");
        failAndPrintHelp(options);
        return;
    }
    List<String> nodes = new ArrayList<>();
    try (ResultSet rs = statement.executeQuery("SELECT nodeid FROM sysnodes")) {
        while (rs.next()) {
            String nodeid = rs.getString(1);
            nodes.add(nodeid);
        }
    }
    println("Restoring tablespace " + newschema + " with leader " + leader + " from file " + inputfile);
    if (!nodes.contains(leader)) {
        println("There is no node with node id '" + leader + "'");
        println("Valid nodes:");
        for (String nodeid : nodes) {
            println("* " + nodeid);
        }
        return;
    }
    try (InputStream fin = wrapStream(file, Files.newInputStream(inputfile));
        InputStream bin = new BufferedInputStream(fin, 16 * 1024 * 1024)) {
        HerdDBConnection hcon = connection.unwrap(HerdDBConnection.class);
        HDBConnection hdbconnection = hcon.getConnection();
        BackupUtils.restoreTableSpace(newschema, leader, hdbconnection, bin, new ProgressListener() {

            @Override
            public void log(String actionType, String message, Map<String, Object> context) {
                println(message);
            }
        });
    }
    println("Restore finished");
}

Example 2 with ProgressListener

use of herddb.backup.ProgressListener in project herddb by diennea.

the class BackupRestoreTest method test_backup_restore_with_deletes.

/**
 * Check that restore a dirty delete doesn't revive a record (it is: a phantom deleted record on a dirty page)
 */
@Test
public void test_backup_restore_with_deletes() throws Exception {
    ServerConfiguration serverconfig_1 = new ServerConfiguration(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_PORT, 7867);
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    /* Disable page compaction (avoid compaction of dirty page) */
    serverconfig_1.set(ServerConfiguration.PROPERTY_FILL_PAGE_THRESHOLD, 0.0D);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath()).set(ServerConfiguration.PROPERTY_PORT, 7868);
    ClientConfiguration client_configuration = new ClientConfiguration(folder.newFolder().toPath());
    client_configuration.set(ClientConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).column("d", ColumnTypes.INTEGER).primaryKey("c").build();
        DBManager manager = server_1.getManager();
        manager.executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.executeUpdate(new InsertStatement(TableSpace.DEFAULT, table.name, RecordSerializer.makeRecord(table, "c", 1, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.executeUpdate(new InsertStatement(TableSpace.DEFAULT, table.name, RecordSerializer.makeRecord(table, "c", 2, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.executeUpdate(new InsertStatement(TableSpace.DEFAULT, table.name, RecordSerializer.makeRecord(table, "c", 3, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.executeUpdate(new InsertStatement(TableSpace.DEFAULT, table.name, RecordSerializer.makeRecord(table, "c", 4, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.checkpoint();
        /* Check that new data isn't in a dirty page  */
        assertEquals(0, manager.getTableSpaceManager(TableSpace.DEFAULT).getTableManager(table.name).getStats().getDirtypages());
        manager.executeUpdate(new DeleteStatement(TableSpace.DEFAULT, table.name, Bytes.from_int(1), null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.executeUpdate(new InsertStatement(TableSpace.DEFAULT, table.name, RecordSerializer.makeRecord(table, "c", 5, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        /* Now we have a dirty page in the checkpoint */
        manager.checkpoint();
        /* Check that a dirty page exists */
        assertEquals(1, manager.getTableSpaceManager(TableSpace.DEFAULT).getTableManager(table.name).getStats().getDirtypages());
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            try (HDBClient client = new HDBClient(client_configuration);
                HDBConnection connection = client.openConnection()) {
                assertEquals(4, connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM t1", Collections.emptyList(), 0, 0, 10).consume().size());
                ByteArrayOutputStream oo = new ByteArrayOutputStream();
                BackupUtils.dumpTableSpace(TableSpace.DEFAULT, 64 * 1024, connection, oo, new ProgressListener() {
                });
                byte[] backupData = oo.toByteArray();
                connection.executeUpdate(TableSpace.DEFAULT, "DELETE FROM t1", 0, false, Collections.emptyList());
                assertEquals(0, connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM t1", Collections.emptyList(), 0, 0, 10).consume().size());
                BackupUtils.restoreTableSpace("newts", server_2.getNodeId(), connection, new ByteArrayInputStream(backupData), new ProgressListener() {
                });
                /* No new insert AND no delete... if 5 it did see the new insert but missed the delete! */
                assertEquals(4, connection.executeScan("newts", "SELECT * FROM newts.t1", Collections.emptyList(), 0, 0, 10).consume().size());
            }
        }
    }
}

Example 3 with ProgressListener

use of herddb.backup.ProgressListener in project herddb by diennea.

the class BackupRestoreTest method test_backup_restore.

@Test
public void test_backup_restore() throws Exception {
    ServerConfiguration serverconfig_1 = new ServerConfiguration(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_PORT, 7867);
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath()).set(ServerConfiguration.PROPERTY_PORT, 7868);
    ClientConfiguration client_configuration = new ClientConfiguration(folder.newFolder().toPath());
    client_configuration.set(ClientConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).column("d", ColumnTypes.INTEGER).primaryKey("c").build();
        Index index = Index.builder().onTable(table).column("d", ColumnTypes.INTEGER).type(Index.TYPE_BRIN).build();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeStatement(new CreateIndexStatement(index), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            try (HDBClient client = new HDBClient(client_configuration);
                HDBConnection connection = client.openConnection()) {
                assertEquals(4, connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM t1", Collections.emptyList(), 0, 0, 10).consume().size());
                ByteArrayOutputStream oo = new ByteArrayOutputStream();
                BackupUtils.dumpTableSpace(TableSpace.DEFAULT, 64 * 1024, connection, oo, new ProgressListener() {
                });
                byte[] backupData = oo.toByteArray();
                connection.executeUpdate(TableSpace.DEFAULT, "DELETE FROM t1", 0, false, Collections.emptyList());
                assertEquals(0, connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM t1", Collections.emptyList(), 0, 0, 10).consume().size());
                BackupUtils.restoreTableSpace("newts", server_2.getNodeId(), connection, new ByteArrayInputStream(backupData), new ProgressListener() {
                });
                assertEquals(4, connection.executeScan("newts", "SELECT * FROM newts.t1", Collections.emptyList(), 0, 0, 10).consume().size());
                assertEquals(1, server_2.getManager().getTableSpaceManager("newts").getIndexesOnTable("t1").size());
            }
        }
    }
}

Example 4 with ProgressListener

use of herddb.backup.ProgressListener in project herddb by diennea.

the class HerdDBCLI method backupTableSpace.

private static void backupTableSpace(final Statement statement, String schema, String file, String suffix, final Connection connection, int dumpfetchsize) throws Exception, SQLException {
    List<String> tablesToDump = new ArrayList<>();
    try (ResultSet rs = statement.executeQuery("SELECT table_name" + " FROM " + schema + ".systables" + " WHERE systemtable='false'")) {
        while (rs.next()) {
            String tablename = rs.getString(1).toLowerCase();
            tablesToDump.add(tablename);
        }
    }
    int dot = file.lastIndexOf('.');
    String ext = "";
    if (dot >= 0) {
        ext = file.substring(dot);
        file = file.substring(0, dot);
    }
    String finalFile = (suffix == null ? file : file + suffix) + ext;
    Path outputfile = Paths.get(finalFile).toAbsolutePath();
    println("Backup tables " + tablesToDump + " from tablespace " + schema + " to " + outputfile);
    try (OutputStream fout = wrapOutputStream(Files.newOutputStream(outputfile, StandardOpenOption.CREATE_NEW), ext);
        SimpleBufferedOutputStream oo = new SimpleBufferedOutputStream(fout, 16 * 1024 * 1024)) {
        HerdDBConnection hcon = connection.unwrap(HerdDBConnection.class);
        HDBConnection hdbconnection = hcon.getConnection();
        BackupUtils.dumpTableSpace(schema, dumpfetchsize, hdbconnection, oo, new ProgressListener() {

            @Override
            public void log(String actionType, String message, Map<String, Object> context) {
                println(message);
            }
        });
    }
    println("Backup finished for tablespace " + schema);
}

Example 5 with ProgressListener

use of herddb.backup.ProgressListener in project herddb by diennea.

the class HugeTableRestoreTest method run.

@Test
public void run() throws Exception {
    generateData(1024);
    performOperations();
    waitForResults();
    // serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PAGES_MEMORY, 5 * 1024 * 1024);
    // serverConfiguration.set(ServerConfiguration.PROPERTY_MEMORY_LIMIT_REFERENCE, 5 * 1024 * 1024);
    restartServer();
    backupRestore(10000, new ProgressListener() {

        @Override
        public void log(String msgType, String message, Map<String, Object> context) {
            System.out.println("PROGESS " + msgType + " -------------------- " + message);
        // if (msgType.equals("sendtabledata")) {
        // try {
        // Thread.sleep(1000);
        // } catch (InterruptedException ee) {
        // throw new RuntimeException(ee);
        // }
        // }
        }
    });
}