Examples with TableManagerStats herddb.core.stats.TableManagerStats used on opensource projects

Example 11 with TableManagerStats

use of herddb.core.stats.TableManagerStats in project herddb by diennea.

the class MultiDMLOnSameRecordTest method testWithFullTableScan.

@Test
public void testWithFullTableScan() throws Exception {
    Path baseDir = folder.newFolder().toPath();
    ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort(baseDir);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 1024 * 1024 / 4);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 1024 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
    serverConfiguration.set(ServerConfiguration.PROPERTY_DATADIR, folder.newFolder().getAbsolutePath());
    serverConfiguration.set(ServerConfiguration.PROPERTY_LOGDIR, folder.newFolder().getAbsolutePath());
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForStandaloneBoot();
        DBManager manager = server.getManager();
        execute(manager, "CREATE TABLE mytable (id string primary key, n1 long, k2 string)", Collections.emptyList());
        ExecutorService threadPool = Executors.newFixedThreadPool(THREADPOLSIZE);
        try {
            List<Future> futures = new ArrayList<>();
            AtomicLong updates = new AtomicLong();
            AtomicLong deletes = new AtomicLong();
            AtomicLong inserts = new AtomicLong();
            AtomicLong duplicatePkErrors = new AtomicLong();
            for (int i = 0; i < TESTSIZE; i++) {
                futures.add(threadPool.submit(new Runnable() {

                    @Override
                    public void run() {
                        Thread.currentThread().setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {

                            @Override
                            public void uncaughtException(Thread t, Throwable e) {
                                e.printStackTrace();
                            }
                        });
                        try {
                            int k = ThreadLocalRandom.current().nextInt(10);
                            long value = ThreadLocalRandom.current().nextInt(100) + 1000;
                            String key = "test_" + k;
                            boolean update = ThreadLocalRandom.current().nextBoolean();
                            boolean insert = ThreadLocalRandom.current().nextBoolean();
                            boolean delete = ThreadLocalRandom.current().nextBoolean();
                            if (update) {
                                updates.incrementAndGet();
                                System.out.println("do " + Thread.currentThread() + " update on " + Bytes.from_string(key));
                                execute(manager, "UPDATE mytable set n1=? WHERE k2=?", Arrays.asList(value, key));
                            } else if (insert) {
                                System.out.println("do " + Thread.currentThread() + " insert on " + Bytes.from_string(key));
                                inserts.incrementAndGet();
                                try {
                                    execute(manager, "INSERT INTO mytable(n1, id, k2) values(?,?,?)", Arrays.asList(value, key, key));
                                } catch (DuplicatePrimaryKeyException ok) {
                                    duplicatePkErrors.incrementAndGet();
                                }
                            } else if (delete) {
                                System.out.println("do " + Thread.currentThread() + " delete on " + Bytes.from_string(key));
                                deletes.incrementAndGet();
                                execute(manager, "DELETE FROM mytable WHERE k2=?", Arrays.asList(key));
                            }
                        } catch (Throwable err) {
                            err.printStackTrace();
                            throw new RuntimeException(err);
                        }
                    }
                }));
            }
            for (Future f : futures) {
                f.get();
            }
            System.out.println("stats::updates:" + updates);
            System.out.println("stats::deletes:" + deletes);
            System.out.println("stats::inserts:" + inserts);
            System.out.println("stats::duplicatePkErrors:" + duplicatePkErrors);
            TableManagerStats stats = server.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("mytable").getStats();
            System.out.println("stats::tablesize:" + stats.getTablesize());
            System.out.println("stats::dirty records:" + stats.getDirtyrecords());
            System.out.println("stats::unload count:" + stats.getUnloadedPagesCount());
            System.out.println("stats::load count:" + stats.getLoadedPagesCount());
            System.out.println("stats::buffers used mem:" + stats.getBuffersUsedMemory());
        } finally {
            threadPool.shutdown();
            threadPool.awaitTermination(1, TimeUnit.MINUTES);
        }
    }
    // restart and recovery
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForTableSpaceBoot(TableSpace.DEFAULT, 300000, true);
    }
}

Example 12 with TableManagerStats

use of herddb.core.stats.TableManagerStats in project herddb by diennea.

the class TableSpaceManager method handleLocalMemoryUsage.

long handleLocalMemoryUsage() {
    long result = 0;
    for (AbstractTableManager tableManager : tables.values()) {
        TableManagerStats stats = tableManager.getStats();
        result += stats.getBuffersUsedMemory();
        result += stats.getKeysUsedMemory();
        result += stats.getDirtyUsedMemory();
    }
    return result;
}

Example 13 with TableManagerStats

use of herddb.core.stats.TableManagerStats in project herddb by diennea.

the class SystablestatsTableManager method buildVirtualRecordList.

@Override
protected Iterable<Record> buildVirtualRecordList(Transaction transaction) {
    List<Table> tables = tableSpaceManager.getAllVisibleTables(transaction);
    List<Record> result = new ArrayList<>();
    for (Table r : tables) {
        AbstractTableManager tableManager = tableSpaceManager.getTableManager(r.name);
        if (tableManager != null && !tableManager.isSystemTable()) {
            TableManagerStats stats = tableManager.getStats();
            result.add(RecordSerializer.makeRecord(table, "tablespace", r.tablespace, "table_name", r.name, "systemtable", r.name.startsWith("sys") ? "true" : "false", "tablesize", stats.getTablesize(), "loadedpages", stats.getLoadedpages(), "loadedpagescount", stats.getLoadedPagesCount(), "unloadedpagescount", stats.getUnloadedPagesCount(), "dirtypages", stats.getDirtypages(), "dirtyrecords", stats.getDirtyrecords(), "maxlogicalpagesize", stats.getMaxLogicalPageSize(), "keysmemory", stats.getKeysUsedMemory(), "buffersmemory", stats.getBuffersUsedMemory()));
        }
    }
    return result;
}

Example 14 with TableManagerStats

use of herddb.core.stats.TableManagerStats in project herddb by diennea.

the class BigTableScanTest method bigTableScan.

@Test
public void bigTableScan() throws Exception {
    int testSize = 5000;
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
        manager.setMaxMemoryReference(128 * 1024);
        manager.setMaxLogicalPageSize(32 * 1024);
        manager.setMaxPKUsedMemory(manager.getMaxLogicalPageSize() * 2);
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        for (int i = 0; i < testSize; i++) {
            InsertStatement insert = new InsertStatement(table.tablespace, table.name, RecordSerializer.makeRecord(table, "id", "k" + i, "name", "testname" + i));
            assertEquals(1, manager.executeUpdate(insert, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        }
        TableManagerStats stats = manager.getTableSpaceManager(table.tablespace).getTableManager(table.name).getStats();
        assertEquals(testSize, stats.getTablesize());
        assertEquals(2, stats.getLoadedpages());
        manager.checkpoint();
        try (DataScanner scan = manager.scan(new ScanStatement(table.tablespace, table, new FullTableScanPredicate()), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)) {
            AtomicInteger count = new AtomicInteger();
            scan.forEach(tuple -> {
                count.incrementAndGet();
            });
            assertEquals(testSize, count.get());
        }
        assertEquals(testSize, stats.getTablesize());
    }
}

Example 15 with TableManagerStats

use of herddb.core.stats.TableManagerStats in project herddb by diennea.

the class CheckpointTest method manyUpdates.

@Test
public void manyUpdates() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmpDir = folder.newFolder("tmpDir").toPath();
    String nodeId = "localhost";
    ServerConfiguration config1 = newServerConfigurationWithAutoPort();
    config1.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 1024 * 1024L);
    config1.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 2 * 1024 * 1024L);
    config1.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 2 * 1024 * 1024L);
    config1.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 10 * 1000L);
    int records = 100;
    int iterations = 1_0000;
    int keylen = 25;
    int strlen = 50;
    Map<String, String> expectedValues = new HashMap<>();
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmpDir, null, config1, null)) {
        // we want frequent checkpoints
        manager.setCheckpointPeriod(10 * 1000L);
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (k1 string, n1 string, primary key(k1))", Collections.emptyList());
        Random random = new Random();
        List<String> keys = new ArrayList<>();
        long tx = TestUtils.beginTransaction(manager, "tblspace1");
        for (int i = 0; i < records; ++i) {
            String key;
            while (true) {
                key = RandomString.getInstance().nextString(keylen);
                if (!keys.contains(key)) {
                    keys.add(key);
                    break;
                }
            }
            String value = RandomString.getInstance().nextString(strlen);
            executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,n1) values(?,?)", Arrays.asList(key, value), new TransactionContext(tx));
            expectedValues.put(key, value);
            if (i % 1000 == 0) {
                System.out.println("commit at " + i);
                TestUtils.commitTransaction(manager, "tblspace1", tx);
                tx = TestUtils.beginTransaction(manager, "tblspace1");
            }
        }
        TestUtils.commitTransaction(manager, "tblspace1", tx);
        System.out.println("database created");
        manager.checkpoint();
        TableManagerStats stats = manager.getTableSpaceManager("tblspace1").getTableManager("tsql").getStats();
        tx = TestUtils.beginTransaction(manager, "tblspace1");
        for (int i = 0; i < iterations; i++) {
            String key = keys.get(random.nextInt(keys.size()));
            String value = RandomString.getInstance().nextString(strlen);
            assertEquals(1, executeUpdate(manager, "UPDATE tblspace1.tsql set n1=? where k1=?", Arrays.asList(value, key, new TransactionContext(tx))).getUpdateCount());
            expectedValues.put(key, value);
            if (random.nextInt(1000) <= 2) {
                System.out.println("checkpoint after " + i + " iterations");
                manager.triggerActivator(ActivatorRunRequest.FULL);
            }
            if (i % 1000 == 0) {
                System.out.println("commit after " + i + " iterations");
                System.out.println("stats: dirtypages:" + stats.getDirtypages() + " unloads:" + stats.getUnloadedPagesCount());
                TestUtils.commitTransaction(manager, "tblspace1", tx);
                tx = TestUtils.beginTransaction(manager, "tblspace1");
            }
        }
        TestUtils.commitTransaction(manager, "tblspace1", tx);
        for (Map.Entry<String, String> expected : expectedValues.entrySet()) {
            try (DataScanner scan = TestUtils.scan(manager, "SELECT n1 FROM tblspace1.tsql where k1=?", Arrays.asList(expected.getKey()))) {
                List<DataAccessor> all = scan.consume();
                assertEquals(1, all.size());
                assertEquals(RawString.of(expected.getValue()), all.get(0).get(0));
            }
        }
        manager.checkpoint();
        for (Map.Entry<String, String> expected : expectedValues.entrySet()) {
            try (DataScanner scan = TestUtils.scan(manager, "SELECT n1 FROM tblspace1.tsql where k1=?", Arrays.asList(expected.getKey()))) {
                List<DataAccessor> all = scan.consume();
                assertEquals(1, all.size());
                assertEquals(RawString.of(expected.getValue()), all.get(0).get(0));
            }
        }
    }
    // reboot
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmpDir, null, config1, null)) {
        // we want frequent checkpoints
        manager.setCheckpointPeriod(10 * 1000L);
        manager.start();
        assertTrue(manager.waitForBootOfLocalTablespaces(60000));
        for (Map.Entry<String, String> expected : expectedValues.entrySet()) {
            try (DataScanner scan = TestUtils.scan(manager, "SELECT n1 FROM tblspace1.tsql where k1=?", Arrays.asList(expected.getKey()))) {
                List<DataAccessor> all = scan.consume();
                assertEquals(1, all.size());
                assertEquals(RawString.of(expected.getValue()), all.get(0).get(0));
            }
        }
    }
}