Examples with ServerConfiguration herddb.server.ServerConfiguration used on opensource projects

Example 81 with ServerConfiguration

use of herddb.server.ServerConfiguration in project herddb by diennea.

the class SecondaryUniqueIndexAccessSuite method secondaryUniqueIndexPrefixScan.

@Test
public void secondaryUniqueIndexPrefixScan() throws Exception {
    String nodeId = "localhost";
    Path tmp = tmpDir.newFolder().toPath();
    ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort(tmp);
    serverConfiguration.set(ServerConfiguration.PROPERTY_READLOCK_TIMEOUT, 3);
    serverConfiguration.set(ServerConfiguration.PROPERTY_WRITELOCK_TIMEOUT, 3);
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), tmp, null, serverConfiguration, NullStatsLogger.INSTANCE)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("n1", ColumnTypes.INTEGER).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        Index index = Index.builder().onTable(table).type(Index.TYPE_HASH).unique(true).column("n1", ColumnTypes.INTEGER).column("name", ColumnTypes.STRING).build();
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('a',1,'n1')", Collections.emptyList());
        // create index, it will be built using existing data
        CreateIndexStatement st3 = new CreateIndexStatement(index);
        manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        // cannot insert another record with '1-n1'
        herddb.utils.TestUtils.assertThrows(UniqueIndexContraintViolationException.class, () -> {
            TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('b',1,'n1')", Collections.emptyList());
        });
        // multiple values
        // it is not a transaction, the first INSERT will succeed
        herddb.utils.TestUtils.assertThrows(UniqueIndexContraintViolationException.class, () -> {
            TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('b',8,'n1'),('c',1,'n1')", Collections.emptyList());
        });
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('d',2,'n2')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('e',3,'n2')", Collections.emptyList());
        // single record UPDATE
        herddb.utils.TestUtils.assertThrows(UniqueIndexContraintViolationException.class, () -> {
            TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=1,name='n1' where id='d'", Collections.emptyList());
        });
        // multi record UPDATE
        herddb.utils.TestUtils.assertThrows(UniqueIndexContraintViolationException.class, () -> {
            TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=1,name='n1' where id='d'", Collections.emptyList());
        });
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1=1", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexPrefixScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(1, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1=8", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexPrefixScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(1, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1=2 and name='n2'", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(1, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1>=1", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            assertNull(scan.getPredicate().getIndexOperation());
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(4, scan1.consume().size());
            }
        }
        // update the index
        TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=10,name='n1' where id='a'", Collections.emptyList());
        TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=1,name='n1' where id='d'", Collections.emptyList());
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1=1", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexPrefixScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                List<DataAccessor> consume = scan1.consume();
                assertEquals(1, consume.size());
                assertEquals(RawString.of("d"), consume.get(0).get("id"));
            }
        }
        // delete, update the index
        TestUtils.executeUpdate(manager, "DELETE FROM tblspace1.t1 where id='d'", Collections.emptyList());
        // enure another record can be stored on the same key
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('o',1,'n1')", Collections.emptyList());
        // test transactions
        long tx = TestUtils.beginTransaction(manager, "tblspace1");
        // insert a new record, and a new index entry, but in transaction, the transacition will hold the index lock on "100-n100"
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('t1',100,'n100')", Collections.emptyList(), new TransactionContext(tx));
        Transaction transaction = manager.getTableSpaceManager("tblspace1").getTransaction(tx);
        assertEquals(1, transaction.locks.get("t1_n1_name").size());
        // delete the same record, the index still hasn't been touched, but we are still holding a lock on "100-n100"
        TestUtils.executeUpdate(manager, "DELETE FROM  tblspace1.t1 where id='t1'", Collections.emptyList(), new TransactionContext(tx));
        // insert a new record again, with the same key in the index, we are still holding the lock on "100-n100"
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('t2',100,'n100')", Collections.emptyList(), new TransactionContext(tx));
        // check that we are going to timeout on lock acquisition from another context (for instance no transaction)
        StatementExecutionException err = herddb.utils.TestUtils.expectThrows(StatementExecutionException.class, () -> {
            TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('t3',100,'n100')", Collections.emptyList());
        });
        assertThat(err.getCause(), instanceOf(LockAcquireTimeoutException.class));
        TestUtils.commitTransaction(manager, "tblspace1", tx);
        // cannot insert another record with "100-n100"
        herddb.utils.TestUtils.assertThrows(UniqueIndexContraintViolationException.class, () -> {
            TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('t3',100,'n100')", Collections.emptyList());
        });
        // again, with transactions, use 200-n200 as index key
        tx = TestUtils.beginTransaction(manager, "tblspace1");
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('s1',200,'n200')", Collections.emptyList(), new TransactionContext(tx));
        // touching the same record, lock has been already held by the transaction
        TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=200 where n1=200", Collections.emptyList(), new TransactionContext(tx));
        TestUtils.commitTransaction(manager, "tblspace1", tx);
        // again, lock the 200-n200 key with a fake update
        tx = TestUtils.beginTransaction(manager, "tblspace1");
        assertEquals(1, TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=200 where n1=200", Collections.emptyList(), new TransactionContext(tx)).getUpdateCount());
        // timeout during an update, lock for 200-n200 is on the transaction, can't perform the same update from another context
        err = herddb.utils.TestUtils.expectThrows(StatementExecutionException.class, () -> {
            TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=200 where n1=200", Collections.emptyList());
        });
        assertThat(err.getCause(), instanceOf(LockAcquireTimeoutException.class));
        assertEquals(1, TestUtils.executeUpdate(manager, "DELETE FROM tblspace1.t1 where n1=200", Collections.emptyList(), new TransactionContext(tx)).getUpdateCount());
        TestUtils.commitTransaction(manager, "tblspace1", tx);
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('s1',200,'n200')", Collections.emptyList());
        // select "FOR UPDATE" does not create locks for the index
        tx = TestUtils.beginTransaction(manager, "tblspace1");
        assertEquals(1, TestUtils.scan(manager, "SELECT * FROM tblspace1.t1 where n1=200 FOR UPDATE", Collections.emptyList(), new TransactionContext(tx)).consumeAndClose().size());
        transaction = manager.getTableSpaceManager("tblspace1").getTransaction(tx);
        assertNull(transaction.locks.get("t1_n1_name"));
        // but it creates a lock for the primary key
        assertEquals(1, transaction.locks.get("t1").size());
        assertEquals(1, TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=200 where n1=200", Collections.emptyList(), new TransactionContext(tx)).getUpdateCount());
        // now we have the lock on the unique index
        assertEquals(1, transaction.locks.get("t1_n1_name").size());
        // still holding the lock for the primary key
        assertEquals(1, transaction.locks.get("t1").size());
        // timeout during an update, lock for 200-n200 is on the transaction, can't perform the same update from another context
        err = herddb.utils.TestUtils.expectThrows(StatementExecutionException.class, () -> {
            TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=200, name='n200' where id='t2'", Collections.emptyList());
        });
        assertThat(err.getCause(), instanceOf(LockAcquireTimeoutException.class));
        TestUtils.commitTransaction(manager, "tblspace1", tx);
    }
}

Example 82 with ServerConfiguration

use of herddb.server.ServerConfiguration in project herddb by diennea.

the class NewPageTest method checkpointEmptyNewPage.

/**
 * Attempt to executre a checkpoint with empty new pages
 */
@Test
public void checkpointEmptyNewPage() throws Exception {
    String nodeId = "localhost";
    ServerConfiguration config1 = newServerConfigurationWithAutoPort();
    /* Smaller pages to avoid too many record creations */
    config1.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null, config1, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (K1 string, S1 string, primary key(k1))", Collections.emptyList());
        manager.checkpoint();
        final TableManager tableManager = (TableManager) manager.getTableSpaceManager("tblspace1").getTableManager("tsql");
        final Table table = tableManager.getTable();
        int records = 0;
        /*
             * Fill table until there are some loaded page (not only kept in memory and unknown to page
             * replacement policy).
             */
        while (tableManager.getLoadedPages().size() < 2) {
            Bytes key = Bytes.from_string("ins_key_" + records++);
            assertEquals(1, manager.executeUpdate(new InsertStatement(table.tablespace, table.name, new Record(key, key)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        }
        /* Checks that every page is still writable and filled */
        for (DataPage page : tableManager.getLoadedPages()) {
            assertTrue(page.writable);
            assertFalse(page.immutable);
            assertFalse(page.isEmpty());
            assertNotEquals(0, page.getUsedMemory());
        }
        /* Fully remove all the data, existing (still mutable) pages should be empty now */
        for (int i = 0; i < records; ++i) {
            Bytes key = Bytes.from_string("ins_key_" + i);
            assertEquals(1, manager.executeUpdate(new DeleteStatement(table.tablespace, table.name, key, null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        }
        /* Checks that every page is still writable and empty */
        for (DataPage page : tableManager.getLoadedPages()) {
            assertTrue(page.writable);
            assertFalse(page.immutable);
            assertTrue(page.isEmpty());
            assertEquals(0, page.getUsedMemory());
        }
        manager.checkpoint();
        /* Expect just one (empty) new page remaining */
        assertEquals(1, tableManager.getLoadedPages().size());
    }
}

Example 83 with ServerConfiguration

use of herddb.server.ServerConfiguration in project herddb by diennea.

the class NewPageTest method unloadEmptyNewPage.

/**
 * Attempt to unload an empty new page from memory
 */
@Test
public void unloadEmptyNewPage() throws Exception {
    String nodeId = "localhost";
    ServerConfiguration config1 = newServerConfigurationWithAutoPort();
    /* Smaller pages to avoid too many record creations */
    config1.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null, config1, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (K1 string, S1 string, primary key(k1))", Collections.emptyList());
        manager.checkpoint();
        final TableManager tableManager = (TableManager) manager.getTableSpaceManager("tblspace1").getTableManager("tsql");
        final Table table = tableManager.getTable();
        int records = 0;
        /*
             * Fill table until there are some loaded page (not only kept in memory and unknown to page
             * replacement policy).
             */
        while (tableManager.getLoadedPages().size() < 2) {
            Bytes key = Bytes.from_string("ins_key_" + records++);
            assertEquals(1, manager.executeUpdate(new InsertStatement(table.tablespace, table.name, new Record(key, key)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        }
        /* Checks that every page is still writable and filled */
        for (DataPage page : tableManager.getLoadedPages()) {
            assertTrue(page.writable);
            assertFalse(page.immutable);
            assertFalse(page.isEmpty());
            assertNotEquals(0, page.getUsedMemory());
        }
        /* Fully remove all the data, existing (still mutable) pages should be empty now */
        for (int i = 0; i < records; ++i) {
            Bytes key = Bytes.from_string("ins_key_" + i);
            assertEquals(1, manager.executeUpdate(new DeleteStatement(table.tablespace, table.name, key, null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        }
        /* Checks that every page is still writable and empty */
        for (DataPage page : tableManager.getLoadedPages()) {
            assertTrue(page.writable);
            assertFalse(page.immutable);
            assertTrue(page.isEmpty());
            assertEquals(0, page.getUsedMemory());
        }
        /* Expect that every empty new page remains */
        assertEquals(2, tableManager.getLoadedPages().size());
        /* Attempt to unload new pages known to page replacement policy */
        boolean unloadedAny = false;
        for (DataPage page : tableManager.getLoadedPages()) {
            if (manager.getMemoryManager().getDataPageReplacementPolicy().remove(page)) {
                unloadedAny = true;
                tableManager.unload(page.pageId);
            }
        }
        /* Ensure that some page was unloaded */
        assertTrue(unloadedAny);
    }
}

Example 84 with ServerConfiguration

use of herddb.server.ServerConfiguration in project herddb by diennea.

the class ChangeDataCapture method start.

/**
 * Bootstrap the procedure.
 * @throws Exception
 */
public void start() throws Exception {
    zookeeperMetadataStorageManager = buildMetadataStorageManager(configuration);
    manager = new BookkeeperCommitLogManager(zookeeperMetadataStorageManager, new ServerConfiguration(), NullStatsLogger.INSTANCE);
    manager.start();
}

Example 85 with ServerConfiguration

use of herddb.server.ServerConfiguration in project herddb by diennea.

the class MultiDMLOnSameRecordTest method testWithFullTableScan.

@Test
public void testWithFullTableScan() throws Exception {
    Path baseDir = folder.newFolder().toPath();
    ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort(baseDir);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 1024 * 1024 / 4);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 1024 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
    serverConfiguration.set(ServerConfiguration.PROPERTY_DATADIR, folder.newFolder().getAbsolutePath());
    serverConfiguration.set(ServerConfiguration.PROPERTY_LOGDIR, folder.newFolder().getAbsolutePath());
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForStandaloneBoot();
        DBManager manager = server.getManager();
        execute(manager, "CREATE TABLE mytable (id string primary key, n1 long, k2 string)", Collections.emptyList());
        ExecutorService threadPool = Executors.newFixedThreadPool(THREADPOLSIZE);
        try {
            List<Future> futures = new ArrayList<>();
            AtomicLong updates = new AtomicLong();
            AtomicLong deletes = new AtomicLong();
            AtomicLong inserts = new AtomicLong();
            AtomicLong duplicatePkErrors = new AtomicLong();
            for (int i = 0; i < TESTSIZE; i++) {
                futures.add(threadPool.submit(new Runnable() {

                    @Override
                    public void run() {
                        Thread.currentThread().setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {

                            @Override
                            public void uncaughtException(Thread t, Throwable e) {
                                e.printStackTrace();
                            }
                        });
                        try {
                            int k = ThreadLocalRandom.current().nextInt(10);
                            long value = ThreadLocalRandom.current().nextInt(100) + 1000;
                            String key = "test_" + k;
                            boolean update = ThreadLocalRandom.current().nextBoolean();
                            boolean insert = ThreadLocalRandom.current().nextBoolean();
                            boolean delete = ThreadLocalRandom.current().nextBoolean();
                            if (update) {
                                updates.incrementAndGet();
                                System.out.println("do " + Thread.currentThread() + " update on " + Bytes.from_string(key));
                                execute(manager, "UPDATE mytable set n1=? WHERE k2=?", Arrays.asList(value, key));
                            } else if (insert) {
                                System.out.println("do " + Thread.currentThread() + " insert on " + Bytes.from_string(key));
                                inserts.incrementAndGet();
                                try {
                                    execute(manager, "INSERT INTO mytable(n1, id, k2) values(?,?,?)", Arrays.asList(value, key, key));
                                } catch (DuplicatePrimaryKeyException ok) {
                                    duplicatePkErrors.incrementAndGet();
                                }
                            } else if (delete) {
                                System.out.println("do " + Thread.currentThread() + " delete on " + Bytes.from_string(key));
                                deletes.incrementAndGet();
                                execute(manager, "DELETE FROM mytable WHERE k2=?", Arrays.asList(key));
                            }
                        } catch (Throwable err) {
                            err.printStackTrace();
                            throw new RuntimeException(err);
                        }
                    }
                }));
            }
            for (Future f : futures) {
                f.get();
            }
            System.out.println("stats::updates:" + updates);
            System.out.println("stats::deletes:" + deletes);
            System.out.println("stats::inserts:" + inserts);
            System.out.println("stats::duplicatePkErrors:" + duplicatePkErrors);
            TableManagerStats stats = server.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("mytable").getStats();
            System.out.println("stats::tablesize:" + stats.getTablesize());
            System.out.println("stats::dirty records:" + stats.getDirtyrecords());
            System.out.println("stats::unload count:" + stats.getUnloadedPagesCount());
            System.out.println("stats::load count:" + stats.getLoadedPagesCount());
            System.out.println("stats::buffers used mem:" + stats.getBuffersUsedMemory());
        } finally {
            threadPool.shutdown();
            threadPool.awaitTermination(1, TimeUnit.MINUTES);
        }
    }
    // restart and recovery
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForTableSpaceBoot(TableSpace.DEFAULT, 300000, true);
    }
}