Examples with ServerConfiguration herddb.server.ServerConfiguration used on opensource projects

Example 66 with ServerConfiguration

use of herddb.server.ServerConfiguration in project herddb by diennea.

the class CheckpointTest method keepSmallRebuiltInMemory.

/**
 * Rebuild small pages but keep rebuilt pages in memory after checkpoint
 */
@Test
public void keepSmallRebuiltInMemory() throws Exception {
    String nodeId = "localhost";
    ServerConfiguration config = newServerConfigurationWithAutoPort();
    /* Force page compaction */
    final double minFillThreashod = 100.0D;
    config.set(ServerConfiguration.PROPERTY_FILL_PAGE_THRESHOLD, minFillThreashod);
    /* Disable dirty rebuilt */
    config.set(ServerConfiguration.PROPERTY_DIRTY_PAGE_THRESHOLD, 100.0D);
    final long pageSize = 200;
    config.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, pageSize);
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null, config, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (K1 string ,s1 string,n1 int, primary key(k1))", Collections.emptyList());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList("mykey1", "a", Integer.valueOf(1234))).getUpdateCount());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList("mykey2", "a", Integer.valueOf(1234))).getUpdateCount());
        manager.checkpoint();
        TableManager table = (TableManager) manager.getTableSpaceManager("tblspace1").getTableManager("tsql");
        /* Get only current and 2 loaded page (+1 empty page for new records... we need the other one) */
        /* We need 2 small pages or small pages compaction will be avoided */
        Collection<DataPage> pages = table.getLoadedPages();
        assertEquals(3, pages.size());
        DataPage page = pages.stream().filter(dpage -> !dpage.writable).findFirst().get();
        assertNotNull(page);
        /* 1 records in page */
        assertEquals(1, page.size());
        /* No more space for other records */
        long avgRecordSize = page.getUsedMemory() / page.size();
        assertTrue(page.getUsedMemory() + avgRecordSize > pageSize);
        /* No more space in page BUT consider the page as small (little hacky) */
        double compactionThresholdSize = minFillThreashod / 100 * pageSize;
        assertTrue(compactionThresholdSize >= page.getUsedMemory());
        /* Force a second checkpoint to attempt to rebuild small page */
        /* Do not unload dirty page */
        /* Now do a checkpoint with an unloaded dirty page */
        manager.checkpoint();
        /* New rebuilt page souldn't be in memory */
        pages = table.getLoadedPages();
        /* 1 page should be empty */
        assertEquals(1, pages.stream().filter(DataPage::isEmpty).count());
        /* 3 pages, one empty, 2 containing old moved records */
        assertEquals(3, pages.size());
    }
}

Example 67 with ServerConfiguration

use of herddb.server.ServerConfiguration in project herddb by diennea.

the class CheckpointTest method keepDirtyTest.

/**
 * Keep dirty pages even after checkpoint
 */
@Test
public void keepDirtyTest() throws Exception {
    String nodeId = "localhost";
    ServerConfiguration config1 = newServerConfigurationWithAutoPort();
    /* Disable page compaction (avoid compaction of dirty page) */
    config1.set(ServerConfiguration.PROPERTY_FILL_PAGE_THRESHOLD, 0.0D);
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null, config1, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (K1 string ,s1 string,n1 int, primary key(k1))", Collections.emptyList());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList("mykey", "a", Integer.valueOf(1234))).getUpdateCount());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList("mykey2", "a", Integer.valueOf(1234))).getUpdateCount());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList("mykey3", "a", Integer.valueOf(1234))).getUpdateCount());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList("mykey4", "a", Integer.valueOf(1234))).getUpdateCount());
        manager.checkpoint();
        /* Dirty a page with few data */
        assertEquals(1, executeUpdate(manager, "UPDATE tblspace1.tsql set s1=? where k1=?", Arrays.asList("b", "mykey4")).getUpdateCount());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList("mykey5", "a", Integer.valueOf(1234))).getUpdateCount());
        manager.checkpoint();
        /* The page is still dirty */
        assertEquals(1, manager.getTableSpaceManager("tblspace1").getTableManager("tsql").getStats().getDirtypages());
    }
}

Example 68 with ServerConfiguration

use of herddb.server.ServerConfiguration in project herddb by diennea.

the class CheckpointTest method checkpointAfterLockTimeout.

@Test
public void checkpointAfterLockTimeout() throws Exception {
    String nodeId = "localhost";
    ServerConfiguration config1 = newServerConfigurationWithAutoPort();
    // 1 second
    config1.set(ServerConfiguration.PROPERTY_WRITELOCK_TIMEOUT, 1);
    config1.set(ServerConfiguration.PROPERTY_READLOCK_TIMEOUT, 1);
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null, config1, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (K1 string ,s1 string,n1 int, primary key(k1))", Collections.emptyList());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList("mykey", "a", Integer.valueOf(1234))).getUpdateCount());
        DMLStatementExecutionResult lockRecord = executeUpdate(manager, "UPDATE tblspace1.tsql set s1=? where k1=?", Arrays.asList("a", "mykey"), TransactionContext.AUTOTRANSACTION_TRANSACTION);
        long tx = lockRecord.transactionId;
        assertTrue(tx > 0);
        assertEquals(1, lockRecord.getUpdateCount());
        // holding a lock on the record, but not the checkpoint lock
        manager.checkpoint();
        StatementExecutionException err = herddb.utils.TestUtils.expectThrows(herddb.model.StatementExecutionException.class, () -> {
            // this update will timeout, lock is already held by the transaction
            executeUpdate(manager, "UPDATE tblspace1.tsql set s1=? where k1=?", Arrays.asList("a", "mykey"), TransactionContext.NO_TRANSACTION);
        });
        assertEquals("timed out acquiring lock for write", err.getCause().getMessage());
        manager.checkpoint();
        err = herddb.utils.TestUtils.expectThrows(herddb.model.StatementExecutionException.class, () -> {
            // this select in a transaction will timeout, write lock is already held by the transaction
            scanKeepReadLocks(manager, "SELECT * FROM tblspace1.tsql", Arrays.asList("mykey"), TransactionContext.AUTOTRANSACTION_TRANSACTION);
        });
        assertEquals("timedout trying to read lock", err.getCause().getMessage());
        manager.checkpoint();
    }
}

Example 69 with ServerConfiguration

use of herddb.server.ServerConfiguration in project herddb by diennea.

the class CheckpointTest method doNotKeepDirtyRebuiltInMemory.

/**
 * Rebuild dirty pages but don't keep rebuilt pages in memory after checkpoint
 */
@Test
public void doNotKeepDirtyRebuiltInMemory() throws Exception {
    String nodeId = "localhost";
    ServerConfiguration config = newServerConfigurationWithAutoPort();
    /* Disable page compaction (avoid compaction of dirty page) */
    config.set(ServerConfiguration.PROPERTY_FILL_PAGE_THRESHOLD, 0.0D);
    /* Force dirty rebuilt */
    config.set(ServerConfiguration.PROPERTY_DIRTY_PAGE_THRESHOLD, 0.0D);
    final long pageSize = 350;
    config.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, pageSize);
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null, config, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (K1 string ,s1 string,n1 int, primary key(k1))", Collections.emptyList());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList("mykey", "a", Integer.valueOf(1234))).getUpdateCount());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList("mykey2", "a", Integer.valueOf(1234))).getUpdateCount());
        manager.checkpoint();
        TableManager table = (TableManager) manager.getTableSpaceManager("tblspace1").getTableManager("tsql");
        /* Get only current and loaded page (+1 empty page for new records... we need the other one) */
        Collection<DataPage> pages = table.getLoadedPages();
        assertEquals(2, pages.size());
        DataPage page = pages.stream().filter(dpage -> !dpage.writable).findFirst().get();
        assertNotNull(page);
        /* 2 records in page */
        assertEquals(2, page.size());
        /* No more space for other records */
        long avgRecordSize = page.getUsedMemory() / page.size();
        assertTrue(page.getUsedMemory() + avgRecordSize > pageSize);
        /* Dirty a page with few data */
        assertEquals(1, executeUpdate(manager, "UPDATE tblspace1.tsql set s1=? where k1=?", Arrays.asList("b", "mykey")).getUpdateCount());
        /*
             * Add new record to fill working page (2 record per page, new "updated" and current insert. We
             * expect remaining dirty record on a new page unloaded)
             */
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList("mykey3", "a", Integer.valueOf(1234))).getUpdateCount());
        /* Force page unload after dirty it */
        /* Remove from page replacement policy */
        manager.getMemoryManager().getDataPageReplacementPolicy().remove(page);
        /* Fully unload the page */
        table.unload(page.pageId);
        /* Now do a checkpoint with an unloaded dirty page */
        manager.checkpoint();
        /* New rebuilt page souldn't be in memory */
        pages = table.getLoadedPages();
        /* 1 page sould be empty */
        assertEquals(1, pages.stream().filter(DataPage::isEmpty).count());
        /* 2 pages, one empty and one contains only moved record and the new record */
        assertEquals(2, pages.size());
    }
}

Example 70 with ServerConfiguration

use of herddb.server.ServerConfiguration in project herddb by diennea.

the class CheckpointTest method manyUpdates.

@Test
public void manyUpdates() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmpDir = folder.newFolder("tmpDir").toPath();
    String nodeId = "localhost";
    ServerConfiguration config1 = newServerConfigurationWithAutoPort();
    config1.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 1024 * 1024L);
    config1.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 2 * 1024 * 1024L);
    config1.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 2 * 1024 * 1024L);
    config1.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 10 * 1000L);
    int records = 100;
    int iterations = 1_0000;
    int keylen = 25;
    int strlen = 50;
    Map<String, String> expectedValues = new HashMap<>();
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmpDir, null, config1, null)) {
        // we want frequent checkpoints
        manager.setCheckpointPeriod(10 * 1000L);
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (k1 string, n1 string, primary key(k1))", Collections.emptyList());
        Random random = new Random();
        List<String> keys = new ArrayList<>();
        long tx = TestUtils.beginTransaction(manager, "tblspace1");
        for (int i = 0; i < records; ++i) {
            String key;
            while (true) {
                key = RandomString.getInstance().nextString(keylen);
                if (!keys.contains(key)) {
                    keys.add(key);
                    break;
                }
            }
            String value = RandomString.getInstance().nextString(strlen);
            executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,n1) values(?,?)", Arrays.asList(key, value), new TransactionContext(tx));
            expectedValues.put(key, value);
            if (i % 1000 == 0) {
                System.out.println("commit at " + i);
                TestUtils.commitTransaction(manager, "tblspace1", tx);
                tx = TestUtils.beginTransaction(manager, "tblspace1");
            }
        }
        TestUtils.commitTransaction(manager, "tblspace1", tx);
        System.out.println("database created");
        manager.checkpoint();
        TableManagerStats stats = manager.getTableSpaceManager("tblspace1").getTableManager("tsql").getStats();
        tx = TestUtils.beginTransaction(manager, "tblspace1");
        for (int i = 0; i < iterations; i++) {
            String key = keys.get(random.nextInt(keys.size()));
            String value = RandomString.getInstance().nextString(strlen);
            assertEquals(1, executeUpdate(manager, "UPDATE tblspace1.tsql set n1=? where k1=?", Arrays.asList(value, key, new TransactionContext(tx))).getUpdateCount());
            expectedValues.put(key, value);
            if (random.nextInt(1000) <= 2) {
                System.out.println("checkpoint after " + i + " iterations");
                manager.triggerActivator(ActivatorRunRequest.FULL);
            }
            if (i % 1000 == 0) {
                System.out.println("commit after " + i + " iterations");
                System.out.println("stats: dirtypages:" + stats.getDirtypages() + " unloads:" + stats.getUnloadedPagesCount());
                TestUtils.commitTransaction(manager, "tblspace1", tx);
                tx = TestUtils.beginTransaction(manager, "tblspace1");
            }
        }
        TestUtils.commitTransaction(manager, "tblspace1", tx);
        for (Map.Entry<String, String> expected : expectedValues.entrySet()) {
            try (DataScanner scan = TestUtils.scan(manager, "SELECT n1 FROM tblspace1.tsql where k1=?", Arrays.asList(expected.getKey()))) {
                List<DataAccessor> all = scan.consume();
                assertEquals(1, all.size());
                assertEquals(RawString.of(expected.getValue()), all.get(0).get(0));
            }
        }
        manager.checkpoint();
        for (Map.Entry<String, String> expected : expectedValues.entrySet()) {
            try (DataScanner scan = TestUtils.scan(manager, "SELECT n1 FROM tblspace1.tsql where k1=?", Arrays.asList(expected.getKey()))) {
                List<DataAccessor> all = scan.consume();
                assertEquals(1, all.size());
                assertEquals(RawString.of(expected.getValue()), all.get(0).get(0));
            }
        }
    }
    // reboot
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmpDir, null, config1, null)) {
        // we want frequent checkpoints
        manager.setCheckpointPeriod(10 * 1000L);
        manager.start();
        assertTrue(manager.waitForBootOfLocalTablespaces(60000));
        for (Map.Entry<String, String> expected : expectedValues.entrySet()) {
            try (DataScanner scan = TestUtils.scan(manager, "SELECT n1 FROM tblspace1.tsql where k1=?", Arrays.asList(expected.getKey()))) {
                List<DataAccessor> all = scan.consume();
                assertEquals(1, all.size());
                assertEquals(RawString.of(expected.getValue()), all.get(0).get(0));
            }
        }
    }
}