Examples with FileDataStorageManager herddb.file.FileDataStorageManager used on opensource projects

Example 41 with FileDataStorageManager

use of herddb.file.FileDataStorageManager in project herddb by diennea.

the class DropOldPagesTest method test.

@Test
public void test() throws Exception {
    FileDataStorageManager fileDataStorageManager = (FileDataStorageManager) dataStorageManager;
    {
        Record record = new Record(Bytes.from_string("key1"), Bytes.from_string("0"));
        InsertStatement st = new InsertStatement(tableSpace, tableName, record);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
    }
    String tableUuid = manager.getTableSpaceManager(tableSpace).getTableManager(tableName).getTable().uuid;
    assertEquals(0, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
    assertEquals(0, fileDataStorageManager.getTablePageFiles(tableSpaceUUID, tableUuid).size());
    manager.checkpoint();
    assertEquals(1, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
    assertEquals(1, fileDataStorageManager.getTablePageFiles(tableSpaceUUID, tableUuid).size());
    {
        Record record = new Record(Bytes.from_string("key2"), Bytes.from_string("0"));
        InsertStatement st = new InsertStatement(tableSpace, tableName, record);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        assertEquals(1, manager.executeUpdate(new DeleteStatement(tableSpace, tableName, Bytes.from_string("key1"), null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
    }
    manager.checkpoint();
    assertEquals(1, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
    assertEquals(1, fileDataStorageManager.getTablePageFiles(tableSpaceUUID, tableUuid).size());
}

Example 42 with FileDataStorageManager

use of herddb.file.FileDataStorageManager in project herddb by diennea.

the class DropTablespaceTest method testCleanPolicyOnDropTable.

@Test
public void testCleanPolicyOnDropTable() throws Exception {
    String nodeId = "localhost";
    ServerConfiguration serverConfig = newServerConfigurationWithAutoPort();
    // very low memory
    serverConfig.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 2_000_000);
    serverConfig.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 2_000_000);
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    try (DBManager manager = new DBManager(nodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new MemoryCommitLogManager(false), tmoDir, null, serverConfig, NullStatsLogger.INSTANCE)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (k1 string primary key,n1 int,s1 string)", Collections.emptyList());
        execute(manager, "CREATE TABLE tblspace1.tsql2 (k1 string primary key,n1 int,s1 string)", Collections.emptyList());
        for (int i = 0; i < 10000; i++) {
            execute(manager, "INSERT INTO tblspace1.tsql (k1 ,n1 ,s1 ) values(?,?,?)", Arrays.asList(i, i, "jkhdskfhjdskfhskdjfhksdhfkshfkjhsdlkhflksjhfklshdkljhsdlkf" + "ksdjhfkdsflksdjhfklsdhjflksdhjflksdhjfklsdhjfklsdjhfsd" + "skjhsdlkfjhdslkfjhsdlkfjhsdlkfhjsklhdfkldshjflksjhdflkhsdf" + "jsdklfjhdslkfjhsdlkfhjslkjhfdlksjhdfklshjdflkhslkfdhslkfhslkfhslkdjfhklsdhjflkhsfd" + "jklsdhfksdjhflksdjhflksjhfdlkjhsdlfkhjslkfjhslkdfjhklsdhjfklhsdff" + "lksdjhflkshdjflkhjsdlkfjhsldkfjhslkfdfoo" + i));
        }
        // drop the table, any reference to the pages should be discarded
        execute(manager, "DROP TABLE tblspace1.tsql", Collections.emptyList());
        // https://github.com/diennea/herddb/issues/434
        for (int i = 0; i < 3000; i++) {
            System.out.println("after " + i);
            execute(manager, "INSERT INTO tblspace1.tsql2 (k1 ,n1 ,s1 ) values(?,?,?)", Arrays.asList(i, i, "jkhdskfhjdskfhskdjfhksdhfkshfkjhsdlkhflksjhfklshdkljhsdlkf" + "ksdjhfkdsflksdjhfklsdhjflksdhjflksdhjfklsdhjfklsdjhfsd" + "skjhsdlkfjhdslkfjhsdlkfjhsdlkfhjsklhdfkldshjflksjhdflkhsdf" + "jsdklfjhdslkfjhsdlkfhjslkjhfdlksjhdfklshjdflkhslkfdhslkfhslkfhslkdjfhklsdhjflkhsfd" + "jklsdhfksdjhflksdjhflksjhfdlkjhsdlfkhjslkfjhslkdfjhklsdhjfklhsdff" + "lksdjhflkshdjflkhjsdlkfjhsldkfjhslkfdfoo" + i));
        }
    }
}

Example 43 with FileDataStorageManager

use of herddb.file.FileDataStorageManager in project herddb by diennea.

the class PrimaryIndexPrefixScanTest method validatePrefixScanIndexWorksForValidDataSet.

@Test
public void validatePrefixScanIndexWorksForValidDataSet() throws Exception {
    String nodeId = "localhost";
    Path dataPath = folder.newFolder("data").toPath();
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new FileDataStorageManager(dataPath), new MemoryCommitLogManager(), null, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("n1", ColumnTypes.INTEGER).column("n2", ColumnTypes.INTEGER).column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("n1").primaryKey("n2").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('a',1,5,'r1')", Collections.emptyList());
        // Same first part of the primary key as the record above but 2nd half is different.
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('b',1,6,'r2')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('c',3,6,'r3')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('d',4,7,'r4')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('e',5,5,'r5')", Collections.emptyList());
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT *" + "FROM tblspace1.t1 " + "WHERE n1=1", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof PrimaryIndexPrefixScan);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            List<DataAccessor> rows = scan1.consume();
            // Assert we got 2 rows.
            assertEquals(2, rows.size());
            for (DataAccessor row : rows) {
                if (row.get("name").equals("r1")) {
                    assertEquals(row.get("n1"), 1);
                    assertEquals(row.get("n2"), 5);
                } else {
                    assertEquals(row.get("name"), "r2");
                    assertEquals(row.get("n1"), 1);
                    assertEquals(row.get("n2"), 6);
                }
            }
        }
    }
}

Example 44 with FileDataStorageManager

use of herddb.file.FileDataStorageManager in project herddb by diennea.

the class AutocheckPointTest method autoCheckPointDuringActivityTest.

@Test
public // @Ignore
void autoCheckPointDuringActivityTest() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    try (DBManager manager = new DBManager(nodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (K1 int ,s1 string,n1 int, primary key(k1))", Collections.emptyList());
        for (int i = 0; i < 100; i++) {
            assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList(i, "a", Integer.valueOf(1234))).getUpdateCount());
        }
        manager.checkpoint();
        long lastCheckpont = manager.getLastCheckPointTs();
        // we want to checkpoint very ofter
        manager.setCheckpointPeriod(100);
        Random random = new Random();
        for (int trial = 0; trial < 1000; trial++) {
            int i = random.nextInt(100);
            execute(manager, "UPDATE tblspace1.tsql set s1='b" + trial + "' where k1=?", Arrays.asList(i), TransactionContext.NO_TRANSACTION);
            Thread.sleep(10);
        }
        assertNotEquals(lastCheckpont, manager.getLastCheckPointTs());
    }
}

Example 45 with FileDataStorageManager

use of herddb.file.FileDataStorageManager in project herddb by diennea.

the class TruncateTableSQLTest method truncateTableFileDataStore.

@Test
public void truncateTableFileDataStore() throws Exception {
    String nodeId = "localhost";
    Path dataPath = folder.newFolder("data").toPath();
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new FileDataStorageManager(dataPath), new MemoryCommitLogManager(), null, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (k1 string primary key,n1 int not nul,s1 string)", Collections.emptyList());
        execute(manager, "CREATE INDEX test1 ON tblspace1.tsql (n1)", Collections.emptyList());
        execute(manager, "INSERT INTO tblspace1.tsql (k1,n1) values('a',1)", Collections.emptyList());
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.tsql ", Collections.emptyList())) {
            assertEquals(1, scan.consume().size());
        } catch (TableDoesNotExistException ok) {
        }
        // truncate, table name non case sensitive
        execute(manager, "TRUNCATE TABLE tblspace1.Tsql", Collections.emptyList());
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.tsql ", Collections.emptyList())) {
            assertEquals(0, scan.consume().size());
        } catch (TableDoesNotExistException ok) {
        }
    }
}