Examples with TableManagerStats herddb.core.stats.TableManagerStats used on opensource projects

Example 6 with TableManagerStats

use of herddb.core.stats.TableManagerStats in project herddb by diennea.

the class MultiDMLOnSameRecordTest method testWithIndexSeek.

@Test
public void testWithIndexSeek() throws Exception {
    Path baseDir = folder.newFolder().toPath();
    ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort(baseDir);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 1024 * 1024 / 4);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 1024 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
    serverConfiguration.set(ServerConfiguration.PROPERTY_DATADIR, folder.newFolder().getAbsolutePath());
    serverConfiguration.set(ServerConfiguration.PROPERTY_LOGDIR, folder.newFolder().getAbsolutePath());
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForStandaloneBoot();
        DBManager manager = server.getManager();
        execute(manager, "CREATE TABLE mytable (id string primary key, n1 long, k2 string)", Collections.emptyList());
        execute(manager, "CREATE INDEX tt ON mytable (k2)", Collections.emptyList());
        ExecutorService threadPool = Executors.newFixedThreadPool(THREADPOLSIZE);
        try {
            List<Future> futures = new ArrayList<>();
            AtomicLong updates = new AtomicLong();
            AtomicLong deletes = new AtomicLong();
            AtomicLong inserts = new AtomicLong();
            AtomicLong duplicatePkErrors = new AtomicLong();
            for (int i = 0; i < TESTSIZE; i++) {
                futures.add(threadPool.submit(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            int k = ThreadLocalRandom.current().nextInt(10);
                            long value = ThreadLocalRandom.current().nextInt(100) + 1000;
                            String key = "test_" + k;
                            boolean update = ThreadLocalRandom.current().nextBoolean();
                            boolean insert = ThreadLocalRandom.current().nextBoolean();
                            boolean delete = ThreadLocalRandom.current().nextBoolean();
                            if (update) {
                                updates.incrementAndGet();
                                execute(manager, "UPDATE mytable set n1=? WHERE k2=?", Arrays.asList(value, key));
                            } else if (insert) {
                                inserts.incrementAndGet();
                                try {
                                    execute(manager, "INSERT INTO mytable(n1, id, k2) values(?,?,?)", Arrays.asList(value, key, key));
                                } catch (DuplicatePrimaryKeyException ok) {
                                    duplicatePkErrors.incrementAndGet();
                                }
                            } else if (delete) {
                                deletes.incrementAndGet();
                                execute(manager, "DELETE FROM mytable WHERE k2=?", Arrays.asList(key));
                            }
                        } catch (Exception err) {
                            throw new RuntimeException(err);
                        }
                    }
                }));
            }
            for (Future f : futures) {
                f.get();
            }
            System.out.println("stats::updates:" + updates);
            System.out.println("stats::deletes:" + deletes);
            System.out.println("stats::inserts:" + inserts);
            System.out.println("stats::duplicatePkErrors:" + duplicatePkErrors);
            TableManagerStats stats = server.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("mytable").getStats();
            System.out.println("stats::tablesize:" + stats.getTablesize());
            System.out.println("stats::dirty records:" + stats.getDirtyrecords());
            System.out.println("stats::unload count:" + stats.getUnloadedPagesCount());
            System.out.println("stats::load count:" + stats.getLoadedPagesCount());
            System.out.println("stats::buffers used mem:" + stats.getBuffersUsedMemory());
        } finally {
            threadPool.shutdown();
            threadPool.awaitTermination(1, TimeUnit.MINUTES);
        }
    }
    // restart and recovery
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForTableSpaceBoot(TableSpace.DEFAULT, 300000, true);
    }
}

Example 7 with TableManagerStats

use of herddb.core.stats.TableManagerStats in project herddb by diennea.

the class MultiDMLOnSameRecordTest method testWithPrimaryKeyIndexSeek.

@Test
public void testWithPrimaryKeyIndexSeek() throws Exception {
    Path baseDir = folder.newFolder().toPath();
    ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort(baseDir);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 1024 * 1024 / 4);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 1024 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
    serverConfiguration.set(ServerConfiguration.PROPERTY_DATADIR, folder.newFolder().getAbsolutePath());
    serverConfiguration.set(ServerConfiguration.PROPERTY_LOGDIR, folder.newFolder().getAbsolutePath());
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForStandaloneBoot();
        DBManager manager = server.getManager();
        execute(manager, "CREATE TABLE mytable (id string primary key, n1 long, n2 integer)", Collections.emptyList());
        ExecutorService threadPool = Executors.newFixedThreadPool(THREADPOLSIZE);
        try {
            List<Future> futures = new ArrayList<>();
            AtomicLong updates = new AtomicLong();
            AtomicLong deletes = new AtomicLong();
            AtomicLong inserts = new AtomicLong();
            AtomicLong duplicatePkErrors = new AtomicLong();
            for (int i = 0; i < TESTSIZE; i++) {
                futures.add(threadPool.submit(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            int k = ThreadLocalRandom.current().nextInt(10);
                            long value = ThreadLocalRandom.current().nextInt(100) + 1000;
                            String key = "test_" + k;
                            boolean update = ThreadLocalRandom.current().nextBoolean();
                            boolean insert = ThreadLocalRandom.current().nextBoolean();
                            boolean delete = ThreadLocalRandom.current().nextBoolean();
                            if (update) {
                                System.err.println("do " + Thread.currentThread() + " update on " + Bytes.from_string(key));
                                updates.incrementAndGet();
                                execute(manager, "UPDATE mytable set n1=? WHERE id=?", Arrays.asList(value, key));
                            } else if (insert) {
                                System.err.println("do " + Thread.currentThread() + " insert on " + Bytes.from_string(key));
                                inserts.incrementAndGet();
                                try {
                                    execute(manager, "INSERT INTO mytable(n1, id) values(?,?)", Arrays.asList(value, key));
                                } catch (DuplicatePrimaryKeyException ok) {
                                    duplicatePkErrors.incrementAndGet();
                                }
                            } else if (delete) {
                                System.err.println("do " + Thread.currentThread() + " delete on " + Bytes.from_string(key));
                                deletes.incrementAndGet();
                                execute(manager, "DELETE FROM mytable WHERE id=?", Arrays.asList(key));
                            }
                        } catch (Exception err) {
                            throw new RuntimeException(err);
                        }
                    }
                }));
            }
            for (Future f : futures) {
                f.get();
            }
            System.out.println("stats::updates:" + updates);
            System.out.println("stats::deletes:" + deletes);
            System.out.println("stats::inserts:" + inserts);
            System.out.println("stats::duplicatePkErrors:" + duplicatePkErrors);
            TableManagerStats stats = server.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("mytable").getStats();
            System.out.println("stats::tablesize:" + stats.getTablesize());
            System.out.println("stats::dirty records:" + stats.getDirtyrecords());
            System.out.println("stats::unload count:" + stats.getUnloadedPagesCount());
            System.out.println("stats::load count:" + stats.getLoadedPagesCount());
            System.out.println("stats::buffers used mem:" + stats.getBuffersUsedMemory());
        } finally {
            threadPool.shutdown();
            threadPool.awaitTermination(1, TimeUnit.MINUTES);
        }
    }
    // restart and recovery
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForTableSpaceBoot(TableSpace.DEFAULT, 300000, true);
    }
}

Example 8 with TableManagerStats

use of herddb.core.stats.TableManagerStats in project herddb by diennea.

the class SystablestatsTableManager method buildVirtualRecordList.

@Override
protected Iterable<Record> buildVirtualRecordList() {
    List<Table> tables = tableSpaceManager.getAllCommittedTables();
    List<Record> result = new ArrayList<>();
    for (Table r : tables) {
        AbstractTableManager tableManager = tableSpaceManager.getTableManager(r.name);
        if (tableManager != null && !tableManager.isSystemTable()) {
            TableManagerStats stats = tableManager.getStats();
            result.add(RecordSerializer.makeRecord(table, "tablespace", r.tablespace, "table_name", r.name, "systemtable", r.name.startsWith("sys") ? "true" : "false", "tablesize", stats.getTablesize(), "loadedpages", stats.getLoadedpages(), "loadedpagescount", stats.getLoadedPagesCount(), "unloadedpagescount", stats.getUnloadedPagesCount(), "dirtypages", stats.getDirtypages(), "dirtyrecords", stats.getDirtyrecords(), "maxlogicalpagesize", stats.getMaxLogicalPageSize(), "keysmemory", stats.getKeysUsedMemory(), "buffersmemory", stats.getBuffersUsedMemory()));
        }
    }
    return result;
}

Example 9 with TableManagerStats

use of herddb.core.stats.TableManagerStats in project herddb by diennea.

the class DirectMultipleConcurrentUpdatesTest method performTest.

private void performTest(boolean useTransactions, long checkPointPeriod, boolean withIndexes) throws Exception {
    Path baseDir = folder.newFolder().toPath();
    ServerConfiguration serverConfiguration = new ServerConfiguration(baseDir);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 1024 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 1024 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, checkPointPeriod);
    serverConfiguration.set(ServerConfiguration.PROPERTY_DATADIR, folder.newFolder().getAbsolutePath());
    serverConfiguration.set(ServerConfiguration.PROPERTY_LOGDIR, folder.newFolder().getAbsolutePath());
    ConcurrentHashMap<String, Long> expectedValue = new ConcurrentHashMap<>();
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForStandaloneBoot();
        DBManager manager = server.getManager();
        execute(manager, "CREATE TABLE mytable (id string primary key, n1 long, n2 integer)", Collections.emptyList());
        if (withIndexes) {
            execute(manager, "CREATE INDEX theindex ON mytable (n1 long)", Collections.emptyList());
        }
        long tx = TestUtils.beginTransaction(manager, TableSpace.DEFAULT);
        for (int i = 0; i < TABLESIZE; i++) {
            TestUtils.executeUpdate(manager, "INSERT INTO mytable (id,n1,n2) values(?,?,?)", Arrays.asList("test_" + i, 1, 2), new TransactionContext(tx));
            expectedValue.put("test_" + i, 1L);
        }
        TestUtils.commitTransaction(manager, TableSpace.DEFAULT, tx);
        ExecutorService threadPool = Executors.newFixedThreadPool(THREADPOLSIZE);
        try {
            List<Future> futures = new ArrayList<>();
            AtomicLong updates = new AtomicLong();
            AtomicLong skipped = new AtomicLong();
            AtomicLong gets = new AtomicLong();
            for (int i = 0; i < TABLESIZE * MULTIPLIER; i++) {
                futures.add(threadPool.submit(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            boolean update = ThreadLocalRandom.current().nextBoolean();
                            int k = ThreadLocalRandom.current().nextInt(TABLESIZE);
                            long value = ThreadLocalRandom.current().nextInt(TABLESIZE);
                            long transactionId;
                            String key = "test_" + k;
                            Long actual = expectedValue.remove(key);
                            if (actual == null) {
                                // another thread working on this entry, skip
                                skipped.incrementAndGet();
                                return;
                            }
                            if (update) {
                                updates.incrementAndGet();
                                DMLStatementExecutionResult updateResult = TestUtils.executeUpdate(manager, "UPDATE mytable set n1=? WHERE id=?", Arrays.asList(value, "test_" + k), new TransactionContext(useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID));
                                long count = updateResult.getUpdateCount();
                                transactionId = updateResult.transactionId;
                                if (count <= 0) {
                                    throw new RuntimeException("not updated ?");
                                }
                            } else {
                                gets.incrementAndGet();
                                DataScanner res = TestUtils.scan(manager, "SELECT * FROM mytable where id=?", Arrays.asList("test_" + k), new TransactionContext(useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID));
                                if (!res.hasNext()) {
                                    throw new RuntimeException("not found?");
                                }
                                res.close();
                                transactionId = res.transactionId;
                                // value did not change actually
                                value = actual;
                            }
                            if (useTransactions) {
                                if (transactionId <= 0) {
                                    throw new RuntimeException("no transaction ?");
                                }
                                commitTransaction(manager, TableSpace.DEFAULT, transactionId);
                            }
                            expectedValue.put(key, value);
                        } catch (Exception err) {
                            throw new RuntimeException(err);
                        }
                    }
                }));
            }
            for (Future f : futures) {
                f.get();
            }
            System.out.println("stats::updates:" + updates);
            System.out.println("stats::get:" + gets);
            assertTrue(updates.get() > 0);
            assertTrue(gets.get() > 0);
            List<String> erroredKeys = new ArrayList<>();
            for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
                List<DataAccessor> records;
                DataAccessor data;
                try (DataScanner res = scan(manager, "SELECT n1 FROM mytable where id=?", Arrays.asList(entry.getKey()))) {
                    records = res.consume();
                    data = records.get(0);
                }
                assertEquals(1, records.size());
                if (!entry.getValue().equals(data.get("n1"))) {
                    System.out.println("expected value " + data.get("n1") + ", but got " + Long.valueOf(entry.getValue()) + " for key " + entry.getKey());
                    erroredKeys.add(entry.getKey());
                }
            }
            assertTrue(erroredKeys.isEmpty());
            TableManagerStats stats = server.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("mytable").getStats();
            System.out.println("stats::tablesize:" + stats.getTablesize());
            System.out.println("stats::dirty records:" + stats.getDirtyrecords());
            System.out.println("stats::unload count:" + stats.getUnloadedPagesCount());
            System.out.println("stats::load count:" + stats.getLoadedPagesCount());
            System.out.println("stats::buffers used mem:" + stats.getBuffersUsedMemory());
            // assertTrue(stats.getUnloadedPagesCount() > 0);
            assertEquals(TABLESIZE, stats.getTablesize());
        } finally {
            threadPool.shutdown();
            threadPool.awaitTermination(1, TimeUnit.MINUTES);
        }
    }
    // restart and recovery
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForStandaloneBoot();
        DBManager manager = server.getManager();
        List<String> erroredKeys = new ArrayList<>();
        for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
            List<DataAccessor> records;
            DataAccessor data;
            try (DataScanner res = scan(manager, "SELECT n1 FROM mytable where id=?", Arrays.asList(entry.getKey()))) {
                records = res.consume();
                data = records.get(0);
            }
            assertEquals(1, records.size());
            if (!entry.getValue().equals(data.get("n1"))) {
                System.out.println("expected value " + data.get("n1") + ", but got " + Long.valueOf(entry.getValue()) + " for key " + entry.getKey());
                erroredKeys.add(entry.getKey());
            }
        }
        assertTrue(erroredKeys.isEmpty());
    }
}

Example 10 with TableManagerStats

use of herddb.core.stats.TableManagerStats in project herddb by diennea.

the class MultipleConcurrentUpdatesTest method performTest.

private void performTest(boolean useTransactions, long checkPointPeriod, boolean withIndexes) throws Exception {
    Path baseDir = folder.newFolder().toPath();
    ServerConfiguration serverConfiguration = new ServerConfiguration(baseDir);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 1024 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 1024 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, checkPointPeriod);
    serverConfiguration.set(ServerConfiguration.PROPERTY_DATADIR, folder.newFolder().getAbsolutePath());
    serverConfiguration.set(ServerConfiguration.PROPERTY_LOGDIR, folder.newFolder().getAbsolutePath());
    ConcurrentHashMap<String, Long> expectedValue = new ConcurrentHashMap<>();
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForStandaloneBoot();
        ClientConfiguration clientConfiguration = new ClientConfiguration(folder.newFolder().toPath());
        try (HDBClient client = new HDBClient(clientConfiguration);
            HDBConnection connection = client.openConnection()) {
            client.setClientSideMetadataProvider(new StaticClientSideMetadataProvider(server));
            long resultCreateTable = connection.executeUpdate(TableSpace.DEFAULT, "CREATE TABLE mytable (id string primary key, n1 long, n2 integer)", 0, false, Collections.emptyList()).updateCount;
            Assert.assertEquals(1, resultCreateTable);
            if (withIndexes) {
                long resultCreateIndex = connection.executeUpdate(TableSpace.DEFAULT, "CREATE INDEX theindex ON mytable (n1 long)", 0, false, Collections.emptyList()).updateCount;
                Assert.assertEquals(1, resultCreateIndex);
            }
            long tx = connection.beginTransaction(TableSpace.DEFAULT);
            for (int i = 0; i < TABLESIZE; i++) {
                connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO mytable (id,n1,n2) values(?,?,?)", tx, false, Arrays.asList("test_" + i, 1, 2));
                expectedValue.put("test_" + i, 1L);
            }
            connection.commitTransaction(TableSpace.DEFAULT, tx);
            ExecutorService threadPool = Executors.newFixedThreadPool(THREADPOLSIZE);
            try {
                List<Future> futures = new ArrayList<>();
                AtomicLong updates = new AtomicLong();
                AtomicLong skipped = new AtomicLong();
                AtomicLong gets = new AtomicLong();
                for (int i = 0; i < TABLESIZE * MULTIPLIER; i++) {
                    futures.add(threadPool.submit(new Runnable() {

                        @Override
                        public void run() {
                            try {
                                boolean update = ThreadLocalRandom.current().nextBoolean();
                                int k = ThreadLocalRandom.current().nextInt(TABLESIZE);
                                long value = ThreadLocalRandom.current().nextInt(TABLESIZE);
                                long transactionId;
                                String key = "test_" + k;
                                Long actual = expectedValue.remove(key);
                                if (actual == null) {
                                    // another thread working on this entry, skip
                                    skipped.incrementAndGet();
                                    return;
                                }
                                if (update) {
                                    updates.incrementAndGet();
                                    DMLResult updateResult = connection.executeUpdate(TableSpace.DEFAULT, "UPDATE mytable set n1=? WHERE id=?", useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID, false, Arrays.asList(value, "test_" + k));
                                    long count = updateResult.updateCount;
                                    transactionId = updateResult.transactionId;
                                    if (count <= 0) {
                                        throw new RuntimeException("not updated ?");
                                    }
                                } else {
                                    gets.incrementAndGet();
                                    GetResult res = connection.executeGet(TableSpace.DEFAULT, "SELECT * FROM mytable where id=?", useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID, Arrays.asList("test_" + k));
                                    if (res.data == null) {
                                        throw new RuntimeException("not found?");
                                    }
                                    if (!res.data.get("n1").equals(actual)) {
                                        throw new RuntimeException("unspected value " + res.data + ", expected: " + actual);
                                    }
                                    transactionId = res.transactionId;
                                    // value did not change actually
                                    value = actual;
                                }
                                if (useTransactions) {
                                    if (transactionId <= 0) {
                                        throw new RuntimeException("no transaction ?");
                                    }
                                    connection.commitTransaction(TableSpace.DEFAULT, transactionId);
                                }
                                expectedValue.put(key, value);
                            } catch (Exception err) {
                                throw new RuntimeException(err);
                            }
                        }
                    }));
                }
                for (Future f : futures) {
                    f.get();
                }
                System.out.println("stats::updates:" + updates);
                System.out.println("stats::get:" + gets);
                System.out.println("stats::skipped:" + skipped);
                assertTrue(updates.get() > 0);
                assertTrue(gets.get() > 0);
                List<String> erroredKeys = new ArrayList<>();
                for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
                    GetResult res = connection.executeGet(TableSpace.DEFAULT, "SELECT n1 FROM mytable where id=?", TransactionContext.NOTRANSACTION_ID, Arrays.asList(entry.getKey()));
                    assertNotNull(res.data);
                    if (!entry.getValue().equals(res.data.get("n1"))) {
                        if (!entry.getValue().equals(res.data.get("n1"))) {
                            System.out.println("expected value " + res.data.get("n1") + ", but got " + Long.valueOf(entry.getValue()) + " for key " + entry.getKey());
                            erroredKeys.add(entry.getKey());
                        }
                    }
                }
                assertTrue(erroredKeys.isEmpty());
                TableManagerStats stats = server.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("mytable").getStats();
                System.out.println("stats::tablesize:" + stats.getTablesize());
                System.out.println("stats::dirty records:" + stats.getDirtyrecords());
                System.out.println("stats::unload count:" + stats.getUnloadedPagesCount());
                System.out.println("stats::load count:" + stats.getLoadedPagesCount());
                System.out.println("stats::buffers used mem:" + stats.getBuffersUsedMemory());
                assertTrue(stats.getUnloadedPagesCount() > 0);
                assertEquals(TABLESIZE, stats.getTablesize());
            } finally {
                threadPool.shutdown();
                threadPool.awaitTermination(1, TimeUnit.MINUTES);
            }
        }
    }
    // restart and recovery
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForStandaloneBoot();
        ClientConfiguration clientConfiguration = new ClientConfiguration(folder.newFolder().toPath());
        try (HDBClient client = new HDBClient(clientConfiguration);
            HDBConnection connection = client.openConnection()) {
            client.setClientSideMetadataProvider(new StaticClientSideMetadataProvider(server));
            for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
                GetResult res = connection.executeGet(TableSpace.DEFAULT, "SELECT n1 FROM mytable where id=?", TransactionContext.NOTRANSACTION_ID, Arrays.asList(entry.getKey()));
                assertNotNull(res.data);
                assertEquals(entry.getValue(), res.data.get("n1"));
            }
        }
    }
}