Search in sources :

Example 6 with TableStatus

use of herddb.storage.TableStatus in project herddb by diennea.

the class BookKeeperDataStorageManager method fullTableScan.

@Override
public void fullTableScan(String tableSpace, String tableUuid, LogSequenceNumber sequenceNumber, FullTableScanConsumer consumer) throws DataStorageManagerException {
    try {
        TableStatus status = getTableStatus(tableSpace, tableUuid, sequenceNumber);
        fullTableScan(tableSpace, tableUuid, status, consumer);
    } catch (HerdDBInternalException err) {
        throw new DataStorageManagerException(err);
    }
}
Also used : DataStorageManagerException(herddb.storage.DataStorageManagerException) HerdDBInternalException(herddb.core.HerdDBInternalException) TableStatus(herddb.storage.TableStatus)

Example 7 with TableStatus

use of herddb.storage.TableStatus in project herddb by diennea.

the class SimpleClusterDisklessTest method test.

@Test
public void test() throws Exception {
    {
        Record record = new Record(Bytes.from_string("key1"), Bytes.from_string("0"));
        InsertStatement st = new InsertStatement(tableSpace, tableName, record);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
    }
    assertEquals(0, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableName));
    manager.checkpoint();
    String tableUuid = manager.getTableSpaceManager(tableSpace).getTableManager(tableName).getTable().uuid;
    assertNotNull(dataStorageManager.readPage(tableSpaceUUID, tableUuid, 1L));
    assertEquals(1, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
    {
        GetResult result = manager.get(new GetStatement(tableSpace, tableName, Bytes.from_string("key1"), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(result.found());
    }
    manager.checkpoint();
    assertEquals(1, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
    {
        Record record = new Record(Bytes.from_string("key1"), Bytes.from_string("5"));
        UpdateStatement st = new UpdateStatement(tableSpace, tableName, record, null);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
    }
    // a new page must be allocated
    manager.checkpoint();
    assertEquals(1, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
    {
        Record record = new Record(Bytes.from_string("key1"), Bytes.from_string("6"));
        UpdateStatement st = new UpdateStatement(tableSpace, tableName, record, null);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
    }
    {
        Record record = new Record(Bytes.from_string("key1"), Bytes.from_string("7"));
        UpdateStatement st = new UpdateStatement(tableSpace, tableName, record, null);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
    }
    // only a new page must be allocated, not two more
    manager.checkpoint();
    assertEquals(1, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
    {
        DeleteStatement st = new DeleteStatement(tableSpace, tableName, Bytes.from_string("key1"), null);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        GetResult result = manager.get(new GetStatement(tableSpace, tableName, Bytes.from_string("key1"), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertFalse(result.found());
    }
    // a delete does not trigger new pages in this case
    manager.checkpoint();
    assertEquals(0, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
    {
        assertEquals(1, manager.executeUpdate(new InsertStatement(tableSpace, tableName, new Record(Bytes.from_string("key2"), Bytes.from_string("50"))), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        assertEquals(1, manager.executeUpdate(new InsertStatement(tableSpace, tableName, new Record(Bytes.from_string("key3"), Bytes.from_string("60"))), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
    }
    manager.checkpoint();
    assertEquals(1, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
    {
        DeleteStatement st = new DeleteStatement(tableSpace, tableName, Bytes.from_string("key2"), null);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
    }
    // a new page, containg the key3 record is needed
    manager.checkpoint();
    assertEquals(1, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
    Holder<TableStatus> _tableStatus = new Holder<>();
    dataStorageManager.fullTableScan(tableSpaceUUID, tableUuid, new FullTableScanConsumer() {

        @Override
        public void acceptTableStatus(TableStatus tableStatus) {
            _tableStatus.value = tableStatus;
        }
    });
    for (long pageId : _tableStatus.value.activePages.keySet()) {
        List<Record> records = dataStorageManager.readPage(tableSpaceUUID, tableUuid, pageId);
        System.out.println("PAGE #" + pageId + " records :" + records);
    }
    assertEquals(1, _tableStatus.value.activePages.size());
}
Also used : UpdateStatement(herddb.model.commands.UpdateStatement) GetResult(herddb.model.GetResult) Holder(herddb.utils.Holder) DeleteStatement(herddb.model.commands.DeleteStatement) InsertStatement(herddb.model.commands.InsertStatement) FullTableScanConsumer(herddb.storage.FullTableScanConsumer) GetStatement(herddb.model.commands.GetStatement) TableStatus(herddb.storage.TableStatus) Record(herddb.model.Record) Test(org.junit.Test)

Example 8 with TableStatus

use of herddb.storage.TableStatus in project herddb by diennea.

the class FileDataStorageManager method fullTableScan.

@Override
public void fullTableScan(String tableSpace, String tableUuid, LogSequenceNumber sequenceNumber, FullTableScanConsumer consumer) throws DataStorageManagerException {
    try {
        TableStatus status = getTableStatus(tableSpace, tableUuid, sequenceNumber);
        fullTableScan(tableSpace, tableUuid, status, consumer);
    } catch (HerdDBInternalException err) {
        throw new DataStorageManagerException(err);
    }
}
Also used : DataStorageManagerException(herddb.storage.DataStorageManagerException) HerdDBInternalException(herddb.core.HerdDBInternalException) TableStatus(herddb.storage.TableStatus)

Example 9 with TableStatus

use of herddb.storage.TableStatus in project herddb by diennea.

the class FileDataStorageManager method tableCheckpoint.

@Override
public List<PostCheckpointAction> tableCheckpoint(String tableSpace, String tableName, TableStatus tableStatus, boolean pin) throws DataStorageManagerException {
    LogSequenceNumber logPosition = tableStatus.sequenceNumber;
    Path dir = getTableDirectory(tableSpace, tableName);
    Path checkpointFile = getTableCheckPointsFile(dir, logPosition);
    try {
        if (Files.isRegularFile(checkpointFile)) {
            TableStatus actualStatus = readTableStatusFromFile(checkpointFile);
            if (actualStatus != null && actualStatus.equals(tableStatus)) {
                LOGGER.log(Level.FINE, "tableCheckpoint " + tableSpace + ", " + tableName + ": " + tableStatus + " (pin:" + pin + ") already saved on file " + checkpointFile);
                return Collections.emptyList();
            }
        }
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    Path parent = getParent(checkpointFile);
    Path checkpointFileTemp = parent.resolve(checkpointFile.getFileName() + ".tmp");
    LOGGER.log(Level.FINE, "tableCheckpoint " + tableSpace + ", " + tableName + ": " + tableStatus + " (pin:" + pin + ") to file " + checkpointFile);
    try (ManagedFile file = ManagedFile.open(checkpointFileTemp, requirefsync);
        SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), COPY_BUFFERS_SIZE);
        XXHash64Utils.HashingOutputStream oo = new XXHash64Utils.HashingOutputStream(buffer);
        ExtendedDataOutputStream dataOutputKeys = new ExtendedDataOutputStream(oo)) {
        // version
        dataOutputKeys.writeVLong(1);
        // flags for future implementations
        dataOutputKeys.writeVLong(0);
        tableStatus.serialize(dataOutputKeys);
        dataOutputKeys.writeLong(oo.hash());
        dataOutputKeys.flush();
        file.sync();
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    try {
        Files.move(checkpointFileTemp, checkpointFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    /* Checkpoint pinning */
    final Map<Long, Integer> pins = pinTableAndGetPages(tableSpace, tableName, tableStatus, pin);
    final Set<LogSequenceNumber> checkpoints = pinTableAndGetCheckpoints(tableSpace, tableName, tableStatus, pin);
    long maxPageId = tableStatus.activePages.keySet().stream().max(Comparator.naturalOrder()).orElse(Long.MAX_VALUE);
    List<PostCheckpointAction> result = new ArrayList<>();
    // we can drop old page files now
    List<Path> pageFiles = getTablePageFiles(tableSpace, tableName);
    for (Path p : pageFiles) {
        long pageId = getPageId(p);
        LOGGER.log(Level.FINEST, "checkpoint file {0} pageId {1}", new Object[] { p.toAbsolutePath(), pageId });
        if (pageId > 0 && !pins.containsKey(pageId) && !tableStatus.activePages.containsKey(pageId) && pageId < maxPageId) {
            LOGGER.log(Level.FINEST, "checkpoint file " + p.toAbsolutePath() + " pageId " + pageId + ". will be deleted after checkpoint end");
            result.add(new DeleteFileAction(tableSpace, tableName, "delete page " + pageId + " file " + p.toAbsolutePath(), p));
        }
    }
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
        for (Path p : stream) {
            if (isTableOrIndexCheckpointsFile(p) && !p.equals(checkpointFile)) {
                TableStatus status = readTableStatusFromFile(p);
                if (logPosition.after(status.sequenceNumber) && !checkpoints.contains(status.sequenceNumber)) {
                    LOGGER.log(Level.FINEST, "checkpoint metadata file " + p.toAbsolutePath() + ". will be deleted after checkpoint end");
                    result.add(new DeleteFileAction(tableSpace, tableName, "delete checkpoint metadata file " + p.toAbsolutePath(), p));
                }
            }
        }
    } catch (IOException err) {
        LOGGER.log(Level.SEVERE, "Could not list table dir " + dir, err);
    }
    return result;
}
Also used : Path(java.nio.file.Path) DataStorageManagerException(herddb.storage.DataStorageManagerException) ArrayList(java.util.ArrayList) LogSequenceNumber(herddb.log.LogSequenceNumber) IOException(java.io.IOException) XXHash64Utils(herddb.utils.XXHash64Utils) ManagedFile(herddb.utils.ManagedFile) ExtendedDataOutputStream(herddb.utils.ExtendedDataOutputStream) PostCheckpointAction(herddb.core.PostCheckpointAction) SimpleBufferedOutputStream(herddb.utils.SimpleBufferedOutputStream) TableStatus(herddb.storage.TableStatus)

Example 10 with TableStatus

use of herddb.storage.TableStatus in project herddb by diennea.

the class FileDataStorageManager method fullTableScan.

@Override
public void fullTableScan(String tableSpace, String tableName, FullTableScanConsumer consumer) throws DataStorageManagerException {
    try {
        TableStatus status = getLatestTableStatus(tableSpace, tableName);
        fullTableScan(tableSpace, tableName, status, consumer);
    } catch (HerdDBInternalException err) {
        throw new DataStorageManagerException(err);
    }
}
Also used : DataStorageManagerException(herddb.storage.DataStorageManagerException) HerdDBInternalException(herddb.core.HerdDBInternalException) TableStatus(herddb.storage.TableStatus)

Aggregations

TableStatus (herddb.storage.TableStatus)16 DataStorageManagerException (herddb.storage.DataStorageManagerException)10 IOException (java.io.IOException)5 HerdDBInternalException (herddb.core.HerdDBInternalException)4 LogSequenceNumber (herddb.log.LogSequenceNumber)4 Record (herddb.model.Record)4 FullTableScanConsumer (herddb.storage.FullTableScanConsumer)4 DataPageMetaData (herddb.core.PageSet.DataPageMetaData)3 ArrayList (java.util.ArrayList)3 HashMap (java.util.HashMap)3 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)3 AtomicLong (java.util.concurrent.atomic.AtomicLong)3 PostCheckpointAction (herddb.core.PostCheckpointAction)2 GetResult (herddb.model.GetResult)2 DeleteStatement (herddb.model.commands.DeleteStatement)2 GetStatement (herddb.model.commands.GetStatement)2 InsertStatement (herddb.model.commands.InsertStatement)2 UpdateStatement (herddb.model.commands.UpdateStatement)2 ExtendedDataOutputStream (herddb.utils.ExtendedDataOutputStream)2 Holder (herddb.utils.Holder)2