Search in sources :

Example 11 with HDBConnection

use of herddb.client.HDBConnection in project herddb by diennea.

the class RetryOnLeaderChangedTest method testSwitchLeaderAndAuthTimeout.

@Test
public void testSwitchLeaderAndAuthTimeout() throws Exception {
    TestStatsProvider statsProvider = new TestStatsProvider();
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    final AtomicBoolean suspendProcessing = new AtomicBoolean(false);
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        try (Server server_2 = new Server(serverconfig_2) {

            @Override
            protected ServerSideConnectionPeer buildPeer(Channel channel) {
                return new ServerSideConnectionPeer(channel, this) {

                    @Override
                    public void requestReceived(Pdu message, Channel channel) {
                        if (suspendProcessing.get()) {
                            LOG.log(Level.INFO, "dropping message type " + message.type + " id " + message.messageId);
                            message.close();
                            return;
                        }
                        super.requestReceived(message, channel);
                    }
                };
            }
        }) {
            server_2.start();
            TestUtils.execute(server_1.getManager(), "CREATE TABLESPACE 'ttt','leader:" + server_2.getNodeId() + "','expectedreplicacount:2'", Collections.emptyList());
            // wait for server_2 to wake up
            for (int i = 0; i < 40; i++) {
                TableSpaceManager tableSpaceManager2 = server_2.getManager().getTableSpaceManager("ttt");
                if (tableSpaceManager2 != null && tableSpaceManager2.isLeader()) {
                    break;
                }
                Thread.sleep(500);
            }
            assertTrue(server_2.getManager().getTableSpaceManager("ttt") != null && server_2.getManager().getTableSpaceManager("ttt").isLeader());
            // wait for server_1 to announce as follower
            waitClusterStatus(server_1.getManager(), server_1.getNodeId(), "follower");
            ClientConfiguration clientConfiguration = new ClientConfiguration();
            clientConfiguration.set(ClientConfiguration.PROPERTY_MODE, ClientConfiguration.PROPERTY_MODE_CLUSTER);
            clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
            clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
            clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
            clientConfiguration.set(ClientConfiguration.PROPERTY_MAX_CONNECTIONS_PER_SERVER, 2);
            clientConfiguration.set(ClientConfiguration.PROPERTY_TIMEOUT, 2000);
            StatsLogger logger = statsProvider.getStatsLogger("ds");
            try (HDBClient client1 = new HDBClient(clientConfiguration, logger) {

                @Override
                public HDBConnection openConnection() {
                    HDBConnection con = new VisibleRouteHDBConnection(this);
                    registerConnection(con);
                    return con;
                }
            }) {
                try (VisibleRouteHDBConnection connection = (VisibleRouteHDBConnection) client1.openConnection()) {
                    // create table and insert data
                    connection.executeUpdate(TableSpace.DEFAULT, "CREATE TABLE ttt.t1(k1 int primary key, n1 int)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                    connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(1,1)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                    assertEquals("server2", connection.getRouteToTableSpace("ttt").getNodeId());
                    // change leader
                    switchLeader(server_1.getNodeId(), server_2.getNodeId(), server_1.getManager());
                    try (VisibleRouteHDBConnection connection2 = (VisibleRouteHDBConnection) client1.openConnection()) {
                        // connection routing still point to old leader (now follower)
                        assertEquals("server2", connection2.getRouteToTableSpace("ttt").getNodeId());
                        // suspend server_2 authentication
                        suspendProcessing.set(true);
                        // attempt an insert with old routing. Suspended autentication generates a timeout
                        // and routing will be reevaluated
                        assertEquals(1, connection2.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(2,2)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList()).updateCount);
                        // right routing to current master
                        assertEquals("server1", connection2.getRouteToTableSpace("ttt").getNodeId());
                        suspendProcessing.set(false);
                    }
                }
            }
        }
    }
}
Also used : TestStatsProvider(org.apache.bookkeeper.test.TestStatsProvider) Pdu(herddb.proto.Pdu) ServerSideConnectionPeer(herddb.server.ServerSideConnectionPeer) StatsLogger(org.apache.bookkeeper.stats.StatsLogger) Server(herddb.server.Server) ServerConfiguration(herddb.server.ServerConfiguration) Channel(herddb.network.Channel) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HDBConnection(herddb.client.HDBConnection) HDBClient(herddb.client.HDBClient) TableSpaceManager(herddb.core.TableSpaceManager) ClientConfiguration(herddb.client.ClientConfiguration) Test(org.junit.Test)

Example 12 with HDBConnection

use of herddb.client.HDBConnection in project herddb by diennea.

the class RetryOnLeaderChangedTest method testKillLeader.

private void testKillLeader(Consumer<HDBConnection> operation) throws Exception {
    TestStatsProvider statsProvider = new TestStatsProvider();
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            TestUtils.execute(server_1.getManager(), "CREATE TABLESPACE 'ttt','leader:" + server_2.getNodeId() + "','expectedreplicacount:2'", Collections.emptyList());
            // wait for server_2 to wake up
            for (int i = 0; i < 40; i++) {
                TableSpaceManager tableSpaceManager2 = server_2.getManager().getTableSpaceManager("ttt");
                if (tableSpaceManager2 != null && tableSpaceManager2.isLeader()) {
                    break;
                }
                Thread.sleep(500);
            }
            assertTrue(server_2.getManager().getTableSpaceManager("ttt") != null && server_2.getManager().getTableSpaceManager("ttt").isLeader());
            // wait for server_1 to announce as follower
            waitClusterStatus(server_1.getManager(), server_1.getNodeId(), "follower");
            ClientConfiguration clientConfiguration = new ClientConfiguration();
            clientConfiguration.set(ClientConfiguration.PROPERTY_MODE, ClientConfiguration.PROPERTY_MODE_CLUSTER);
            clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
            clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
            clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
            clientConfiguration.set(ClientConfiguration.PROPERTY_MAX_OPERATION_RETRY_COUNT, 1000);
            StatsLogger logger = statsProvider.getStatsLogger("ds");
            try (HDBClient client1 = new HDBClient(clientConfiguration, logger)) {
                try (HDBConnection connection = client1.openConnection()) {
                    // create table and insert data
                    connection.executeUpdate(TableSpace.DEFAULT, "CREATE TABLE ttt.t1(k1 int primary key, n1 int)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                    connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(1,1)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                    connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(2,1)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                    connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(3,1)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                    // use client2, so that it opens a connection to current leader
                    try (ScanResultSet scan = connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM ttt.t1", false, Collections.emptyList(), TransactionContext.NOTRANSACTION_ID, 0, 0, true)) {
                        assertEquals(3, scan.consume().size());
                    }
                    // change leader
                    switchLeader(server_1.getNodeId(), null, server_1.getManager());
                    // make server_2 (current leader) die
                    server_2.close();
                    // perform operation, it will eventually succeed, because
                    // the client will automatically wait for the new leader to be up
                    operation.accept(connection);
                }
            }
        }
    }
}
Also used : TestStatsProvider(org.apache.bookkeeper.test.TestStatsProvider) StatsLogger(org.apache.bookkeeper.stats.StatsLogger) HDBConnection(herddb.client.HDBConnection) HDBClient(herddb.client.HDBClient) Server(herddb.server.Server) ServerConfiguration(herddb.server.ServerConfiguration) TableSpaceManager(herddb.core.TableSpaceManager) ScanResultSet(herddb.client.ScanResultSet) ClientConfiguration(herddb.client.ClientConfiguration)

Example 13 with HDBConnection

use of herddb.client.HDBConnection in project herddb by diennea.

the class JAASKerberosTest method test.

@Test
public void test() throws Exception {
    ServerConfiguration serverConfig = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverConfig.set(ServerConfiguration.PROPERTY_HOST, "localhost");
    try (Server server = new Server(serverConfig)) {
        server.start();
        try (HDBClient client = new HDBClient(new ClientConfiguration(folder.newFolder().toPath()));
            HDBConnection connection = client.openConnection()) {
            client.setClientSideMetadataProvider(new StaticClientSideMetadataProvider(server));
            long resultCreateTable = connection.executeUpdate(TableSpace.DEFAULT, "CREATE TABLE mytable (id string primary key, n1 long, n2 integer)", 0, false, true, Collections.emptyList()).updateCount;
            Assert.assertEquals(1, resultCreateTable);
            Assert.assertEquals(1, connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO mytable (id,n1,n2) values(?,?,?)", 0, false, true, Arrays.asList("test_0", 1, 2)).updateCount);
            Map<RawString, Object> newValue = connection.executeUpdate(TableSpace.DEFAULT, "UPDATE mytable set n1= n1+1 where id=?", 0, true, true, Arrays.asList("test_0")).newvalue;
            assertEquals(Long.valueOf(2), newValue.get(RawString.of("n1")));
        }
    } finally {
        System.clearProperty("java.security.auth.login.config");
    }
}
Also used : HDBConnection(herddb.client.HDBConnection) StaticClientSideMetadataProvider(herddb.server.StaticClientSideMetadataProvider) HDBClient(herddb.client.HDBClient) Server(herddb.server.Server) RawString(herddb.utils.RawString) ServerConfiguration(herddb.server.ServerConfiguration) ClientConfiguration(herddb.client.ClientConfiguration) Test(org.junit.Test)

Example 14 with HDBConnection

use of herddb.client.HDBConnection in project herddb by diennea.

the class BasicHerdDBDataSource method getHDBConnection.

private synchronized HDBConnection getHDBConnection() {
    if (connection == null) {
        HDBClient _client = getClient();
        HDBConnection _connection = _client.openConnection();
        _connection.setDiscoverTablespaceFromSql(false);
        connection = _connection;
    }
    return connection;
}
Also used : HDBConnection(herddb.client.HDBConnection) HDBClient(herddb.client.HDBClient)

Example 15 with HDBConnection

use of herddb.client.HDBConnection in project herddb by diennea.

the class DownloadTableSpaceTest method downloadTablespaceTest.

@Test
public void downloadTablespaceTest() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        for (int i = 0; i < 1000; i++) {
            server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", i)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        }
        List<Map<String, Object>> logical_data = new ArrayList<>();
        AtomicBoolean start = new AtomicBoolean();
        try (HDBClient client = new HDBClient(new ClientConfiguration(folder.newFolder().toPath()));
            HDBConnection con = client.openConnection()) {
            client.setClientSideMetadataProvider(new ZookeeperClientSideMetadataProvider(testEnv.getAddress(), testEnv.getTimeout(), testEnv.getPath()));
            CountDownLatch count = new CountDownLatch(1);
            con.dumpTableSpace(TableSpace.DEFAULT, new TableSpaceDumpReceiver() {

                Table table;

                @Override
                public void start(LogSequenceNumber logSequenceNumber) throws DataStorageManagerException {
                    System.out.println("start at " + logSequenceNumber);
                    start.set(true);
                }

                @Override
                public void finish(LogSequenceNumber logSequenceNumber) {
                    System.out.println("finish!");
                    count.countDown();
                }

                @Override
                public void endTable() {
                    System.out.println("endTable");
                    table = null;
                }

                @Override
                public void receiveTableDataChunk(List<Record> records) {
                    // System.out.println("receiveTableDataChunk " + records);
                    for (Record r : records) {
                        Map<String, Object> bean = r.toBean(table);
                        // System.out.println("received:" + bean);
                        logical_data.add(bean);
                    }
                }

                @Override
                public void beginTable(DumpedTableMetadata table, Map<String, Object> stats) {
                    System.out.println("beginTable " + table);
                    this.table = table.table;
                }
            }, 89, false);
            assertTrue(count.await(20, TimeUnit.SECONDS));
            assertEquals(1000, logical_data.size());
            assertTrue(start.get());
        }
    }
}
Also used : DataStorageManagerException(herddb.storage.DataStorageManagerException) ArrayList(java.util.ArrayList) InsertStatement(herddb.model.commands.InsertStatement) DumpedTableMetadata(herddb.backup.DumpedTableMetadata) HDBConnection(herddb.client.HDBConnection) TableSpaceDumpReceiver(herddb.client.TableSpaceDumpReceiver) HDBClient(herddb.client.HDBClient) Record(herddb.model.Record) Table(herddb.model.Table) CreateTableStatement(herddb.model.commands.CreateTableStatement) LogSequenceNumber(herddb.log.LogSequenceNumber) CountDownLatch(java.util.concurrent.CountDownLatch) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ZookeeperClientSideMetadataProvider(herddb.client.ZookeeperClientSideMetadataProvider) Map(java.util.Map) ClientConfiguration(herddb.client.ClientConfiguration) Test(org.junit.Test)

Aggregations

HDBConnection (herddb.client.HDBConnection)48 HDBClient (herddb.client.HDBClient)45 ClientConfiguration (herddb.client.ClientConfiguration)44 Test (org.junit.Test)38 Table (herddb.model.Table)20 CreateTableStatement (herddb.model.commands.CreateTableStatement)20 InsertStatement (herddb.model.commands.InsertStatement)19 ProgressListener (herddb.backup.ProgressListener)15 Server (herddb.server.Server)14 ServerConfiguration (herddb.server.ServerConfiguration)14 ScanResultSet (herddb.client.ScanResultSet)13 ByteArrayInputStream (java.io.ByteArrayInputStream)12 ByteArrayOutputStream (java.io.ByteArrayOutputStream)12 Index (herddb.model.Index)9 CreateIndexStatement (herddb.model.commands.CreateIndexStatement)9 TableSpaceManager (herddb.core.TableSpaceManager)8 HashSet (java.util.HashSet)8 Map (java.util.Map)8 RawString (herddb.utils.RawString)7 Path (java.nio.file.Path)7