Examples with CommitLog herddb.log.CommitLog used on opensource projects

Example 1 with CommitLog

use of herddb.log.CommitLog in project herddb by diennea.

the class FileCommitLogTest method testLog.

@Test
public void testLog() throws Exception {
    FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath(), 64 * 1024 * 1024);
    int writeCount = 0;
    final long _startWrite = System.currentTimeMillis();
    try (CommitLog log = manager.createCommitLog("tt")) {
        log.startWriting();
        for (int i = 0; i < 10_000; i++) {
            log.log(LogEntryFactory.beginTransaction(0), false);
            writeCount++;
        }
    }
    final long _endWrite = System.currentTimeMillis();
    AtomicInteger readCount = new AtomicInteger();
    try (CommitLog log = manager.createCommitLog("tt")) {
        log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

            @Override
            public void accept(LogSequenceNumber t, LogEntry u) {
                readCount.incrementAndGet();
            }
        }, true);
    }
    final long _endRead = System.currentTimeMillis();
    assertEquals(writeCount, readCount.get());
    System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
    System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
}

Example 2 with CommitLog

use of herddb.log.CommitLog in project herddb by diennea.

the class FileCommitLogTest method testLogsynch.

@Test
public void testLogsynch() throws Exception {
    FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath(), 64 * 1024 * 1024);
    int writeCount = 0;
    final long _startWrite = System.currentTimeMillis();
    try (CommitLog log = manager.createCommitLog("tt")) {
        log.startWriting();
        for (int i = 0; i < 100; i++) {
            log.log(LogEntryFactory.beginTransaction(0), true);
            writeCount++;
        }
    }
    final long _endWrite = System.currentTimeMillis();
    AtomicInteger readCount = new AtomicInteger();
    try (CommitLog log = manager.createCommitLog("tt")) {
        log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

            @Override
            public void accept(LogSequenceNumber t, LogEntry u) {
                readCount.incrementAndGet();
            }
        }, true);
    }
    final long _endRead = System.currentTimeMillis();
    assertEquals(writeCount, readCount.get());
    System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
    System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
}

Example 3 with CommitLog

use of herddb.log.CommitLog in project herddb by diennea.

the class DBManager method handleTableSpace.

private void handleTableSpace(TableSpace tableSpace) throws DataStorageManagerException, LogNotAvailableException, MetadataStorageManagerException, DDLException {
    String tableSpaceName = tableSpace.name;
    TableSpaceManager actual_manager = tablesSpaces.get(tableSpaceName);
    if (actual_manager != null && actual_manager.isFailed()) {
        LOGGER.log(Level.INFO, "Tablespace {0} is in 'Failed' status", new Object[] { tableSpaceName, nodeId });
        return;
    }
    if (actual_manager != null && actual_manager.isLeader() && !tableSpace.leaderId.equals(nodeId)) {
        LOGGER.log(Level.SEVERE, "Tablespace {0} leader is no more {1}, it changed to {2}", new Object[] { tableSpaceName, nodeId, tableSpace.leaderId });
        stopTableSpace(tableSpaceName, tableSpace.uuid);
        return;
    }
    if (actual_manager != null && !actual_manager.isLeader() && tableSpace.leaderId.equals(nodeId)) {
        LOGGER.log(Level.SEVERE, "Tablespace {0} need to switch to leadership on node {1}", new Object[] { tableSpaceName, nodeId });
        stopTableSpace(tableSpaceName, tableSpace.uuid);
        return;
    }
    if (tableSpace.replicas.contains(nodeId) && !tablesSpaces.containsKey(tableSpaceName)) {
        LOGGER.log(Level.SEVERE, "Booting tablespace {0} on {1}, uuid {2}", new Object[] { tableSpaceName, nodeId, tableSpace.uuid });
        long _start = System.currentTimeMillis();
        CommitLog commitLog = commitLogManager.createCommitLog(tableSpace.uuid);
        TableSpaceManager manager = new TableSpaceManager(nodeId, tableSpaceName, tableSpace.uuid, metadataStorageManager, dataStorageManager, commitLog, this, false);
        try {
            manager.start();
            LOGGER.log(Level.SEVERE, "Boot success tablespace {0} on {1}, uuid {2}, time {3} ms", new Object[] { tableSpaceName, nodeId, tableSpace.uuid, (System.currentTimeMillis() - _start) + "" });
            tablesSpaces.put(tableSpaceName, manager);
            if (serverConfiguration.getBoolean(ServerConfiguration.PROPERTY_JMX_ENABLE, ServerConfiguration.PROPERTY_JMX_ENABLE_DEFAULT)) {
                JMXUtils.registerTableSpaceManagerStatsMXBean(tableSpaceName, manager.getStats());
            }
        } catch (DataStorageManagerException | LogNotAvailableException | MetadataStorageManagerException | DDLException t) {
            LOGGER.log(Level.SEVERE, "Error Booting tablespace {0} on {1}", new Object[] { tableSpaceName, nodeId });
            LOGGER.log(Level.SEVERE, "Error", t);
            try {
                manager.close();
            } catch (Throwable t2) {
                LOGGER.log(Level.SEVERE, "Other Error", t2);
            }
            throw t;
        }
        return;
    }
    if (tablesSpaces.containsKey(tableSpaceName) && !tableSpace.replicas.contains(nodeId)) {
        LOGGER.log(Level.SEVERE, "Tablespace {0} on {1} is not more in replica list {3}, uuid {2}", new Object[] { tableSpaceName, nodeId, tableSpace.uuid, tableSpace.replicas + "" });
        stopTableSpace(tableSpaceName, tableSpace.uuid);
        return;
    }
    if (tableSpace.replicas.size() < tableSpace.expectedReplicaCount) {
        List<NodeMetadata> nodes = metadataStorageManager.listNodes();
        LOGGER.log(Level.SEVERE, "Tablespace {0} is underreplicated expectedReplicaCount={1}, replicas={2}, nodes={3}", new Object[] { tableSpaceName, tableSpace.expectedReplicaCount, tableSpace.replicas, nodes });
        List<String> availableOtherNodes = nodes.stream().map(n -> {
            return n.nodeId;
        }).filter(n -> {
            return !tableSpace.replicas.contains(n);
        }).collect(Collectors.toList());
        Collections.shuffle(availableOtherNodes);
        LOGGER.log(Level.SEVERE, "Tablespace {0} is underreplicated expectedReplicaCount={1}, replicas={2}, availableOtherNodes={3}", new Object[] { tableSpaceName, tableSpace.expectedReplicaCount, tableSpace.replicas, availableOtherNodes });
        if (!availableOtherNodes.isEmpty()) {
            int countMissing = tableSpace.expectedReplicaCount - tableSpace.replicas.size();
            TableSpace.Builder newTableSpaceBuilder = TableSpace.builder().cloning(tableSpace);
            while (!availableOtherNodes.isEmpty() && countMissing > 0) {
                String node = availableOtherNodes.remove(0);
                newTableSpaceBuilder.replica(node);
            }
            TableSpace newTableSpace = newTableSpaceBuilder.build();
            boolean ok = metadataStorageManager.updateTableSpace(newTableSpace, tableSpace);
            if (!ok) {
                LOGGER.log(Level.SEVERE, "updating tableSpace " + tableSpaceName + " metadata failed");
            }
        }
    }
}

Example 4 with CommitLog

use of herddb.log.CommitLog in project herddb by diennea.

the class MemoryCommitLogManager method createCommitLog.

@Override
public CommitLog createCommitLog(String tableSpace) {
    return new CommitLog() {

        AtomicLong offset = new AtomicLong(-1);

        @Override
        public CommitLogResult log(LogEntry entry, boolean synch) throws LogNotAvailableException {
            if (isHasListeners()) {
                synch = true;
            }
            // NOOP
            entry.serialize();
            LogSequenceNumber logPos = new LogSequenceNumber(1, offset.incrementAndGet());
            notifyListeners(logPos, entry);
            return new CommitLogResult(logPos, !synch);
        }

        @Override
        public LogSequenceNumber getLastSequenceNumber() {
            return new LogSequenceNumber(1, offset.get());
        }

        private volatile boolean closed;

        @Override
        public void close() throws LogNotAvailableException {
            closed = true;
        }

        @Override
        public boolean isFailed() {
            return false;
        }

        @Override
        public boolean isClosed() {
            return closed;
        }

        @Override
        public void recovery(LogSequenceNumber snapshotSequenceNumber, BiConsumer<LogSequenceNumber, LogEntry> consumer, boolean fencing) throws LogNotAvailableException {
        }

        @Override
        public void dropOldLedgers(LogSequenceNumber lastCheckPointSequenceNumber) throws LogNotAvailableException {
        }

        @Override
        public void followTheLeader(LogSequenceNumber skipPast, BiConsumer<LogSequenceNumber, LogEntry> consumer) throws LogNotAvailableException {
        }

        @Override
        public void startWriting() throws LogNotAvailableException {
        }

        @Override
        public void clear() throws LogNotAvailableException {
        }
    };
}

Example 5 with CommitLog

use of herddb.log.CommitLog in project herddb by diennea.

the class FileCommitLogTest method testDiskFullLogMissingFooter.

@Test
public void testDiskFullLogMissingFooter() throws Exception {
    try (FileCommitLogManager manager = new FileCommitLogManager(folder.newFolder().toPath(), 64 * 1024 * 1024)) {
        int writeCount = 0;
        final long _startWrite = System.currentTimeMillis();
        try (CommitLog log = manager.createCommitLog("tt")) {
            log.startWriting();
            for (int i = 0; i < 100; i++) {
                log.log(LogEntryFactory.beginTransaction(0), false);
                writeCount++;
            }
            FileCommitLog fileCommitLog = (FileCommitLog) log;
            // simulate end of disk
            byte[] dummyEntry = LogEntryFactory.beginTransaction(0).serialize();
            // header
            fileCommitLog.getWriter().out.write(ENTRY_START);
            fileCommitLog.getWriter().out.writeLong(0);
            // entry
            fileCommitLog.getWriter().out.write(dummyEntry);
            // missing entry footer
            fileCommitLog.getWriter().out.flush();
        }
        final long _endWrite = System.currentTimeMillis();
        AtomicInteger readCount = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount.incrementAndGet();
                }
            }, true);
        }
        final long _endRead = System.currentTimeMillis();
        assertEquals(writeCount, readCount.get());
        System.out.println("Write time: " + (_endWrite - _startWrite) + " ms");
        System.out.println("Read time: " + (_endRead - _endWrite) + " ms");
        // must be able to read twice
        AtomicInteger readCount2 = new AtomicInteger();
        try (CommitLog log = manager.createCommitLog("tt")) {
            log.recovery(LogSequenceNumber.START_OF_TIME, new BiConsumer<LogSequenceNumber, LogEntry>() {

                @Override
                public void accept(LogSequenceNumber t, LogEntry u) {
                    readCount2.incrementAndGet();
                }
            }, true);
        }
        assertEquals(writeCount, readCount.get());
    }
}