Examples with OLogSequenceNumber com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber used on opensource projects

Example 86 with OLogSequenceNumber

use of com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber in project orientdb by orientechnologies.

the class ReadWriteDiskCacheTest method testCloseFileShouldFlushData.

public void testCloseFileShouldFlushData() throws Exception {
    long fileId = readBuffer.addFile(fileName, writeBuffer);
    OCacheEntry[] entries = new OCacheEntry[4];
    for (int i = 0; i < 4; i++) {
        entries[i] = readBuffer.load(fileId, i, false, writeBuffer, 1, true);
        if (entries[i] == null) {
            entries[i] = readBuffer.allocateNewPage(fileId, writeBuffer, true);
            Assert.assertEquals(entries[i].getPageIndex(), i);
        }
        entries[i].getCachePointer().acquireExclusiveLock();
        entries[i].markDirty();
        ByteBuffer buffer = entries[i].getCachePointer().getSharedBuffer();
        buffer.position(systemOffset);
        buffer.put(new byte[] { (byte) i, 1, 2, seed, 4, 5, 6, (byte) i });
        entries[i].getCachePointer().releaseExclusiveLock();
        readBuffer.release(entries[i], writeBuffer);
    }
    LRUList am = readBuffer.getAm();
    LRUList a1in = readBuffer.getA1in();
    LRUList a1out = readBuffer.getA1out();
    Assert.assertEquals(am.size(), 0);
    Assert.assertEquals(a1out.size(), 0);
    final OByteBufferPool bufferPool = OByteBufferPool.instance();
    for (int i = 0; i < 4; i++) {
        OCacheEntry entry = generateEntry(fileId, i, entries[i].getCachePointer().getSharedBuffer(), bufferPool, false, new OLogSequenceNumber(0, 0));
        Assert.assertEquals(a1in.get(entry.getFileId(), entry.getPageIndex()), entry);
    }
    Assert.assertEquals(writeBuffer.getFilledUpTo(fileId), 4);
    readBuffer.closeFile(fileId, true, writeBuffer);
    for (int i = 0; i < 4; i++) {
        assertFile(i, new byte[] { (byte) i, 1, 2, seed, 4, 5, 6, (byte) i }, new OLogSequenceNumber(0, 0));
    }
}

Example 87 with OLogSequenceNumber

use of com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber in project orientdb by orientechnologies.

the class ReadWriteDiskCacheTest method testLoadAndLockForReadShouldHitCache.

public void testLoadAndLockForReadShouldHitCache() throws Exception {
    long fileId = readBuffer.addFile(fileName, writeBuffer);
    OCacheEntry cacheEntry = readBuffer.load(fileId, 0, false, writeBuffer, 1, true);
    if (cacheEntry == null) {
        cacheEntry = readBuffer.allocateNewPage(fileId, writeBuffer, true);
        Assert.assertEquals(cacheEntry.getPageIndex(), 0);
    }
    readBuffer.release(cacheEntry, writeBuffer);
    LRUList am = readBuffer.getAm();
    LRUList a1in = readBuffer.getA1in();
    LRUList a1out = readBuffer.getA1out();
    Assert.assertEquals(am.size(), 0);
    Assert.assertEquals(a1out.size(), 0);
    final OByteBufferPool bufferPool = OByteBufferPool.instance();
    final OCacheEntry entry = generateEntry(fileId, 0, cacheEntry.getCachePointer().getSharedBuffer(), bufferPool, false, new OLogSequenceNumber(0, 0));
    Assert.assertEquals(a1in.size(), 1);
    Assert.assertEquals(a1in.get(entry.getFileId(), entry.getPageIndex()), entry);
}

Example 88 with OLogSequenceNumber

use of com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber in project orientdb by orientechnologies.

the class ReadWriteDiskCacheTest method testPrefetchPagesInA1inQueue.

public void testPrefetchPagesInA1inQueue() throws Exception {
    final long fileId = readBuffer.addFile(fileName, writeBuffer);
    for (int i = 0; i < 4; i++) {
        OCacheEntry cacheEntry = readBuffer.allocateNewPage(fileId, writeBuffer, true);
        cacheEntry.acquireExclusiveLock();
        try {
            byte[] userData = new byte[userDataSize];
            for (int n = 0; n < userData.length; n++) {
                userData[n] = (byte) (i + 1);
            }
            final ByteBuffer buffer = cacheEntry.getCachePointer().getSharedBuffer();
            buffer.position(systemOffset);
            buffer.put(userData);
            setLsn(buffer, new OLogSequenceNumber(1, i));
            cacheEntry.markDirty();
        } finally {
            cacheEntry.releaseExclusiveLock();
            readBuffer.release(cacheEntry, writeBuffer);
        }
    }
    readBuffer.clear();
    writeBuffer.flush();
    for (int i = 0; i < 4; i++) {
        byte[] userData = new byte[userDataSize];
        for (int n = 0; n < userData.length; n++) {
            userData[n] = (byte) (i + 1);
        }
        assertFile(i, userData, new OLogSequenceNumber(1, i));
    }
    LRUList am = readBuffer.getAm();
    LRUList a1in = readBuffer.getA1in();
    LRUList a1out = readBuffer.getA1out();
    Assert.assertEquals(am.size(), 0);
    Assert.assertEquals(a1out.size(), 0);
    Assert.assertEquals(a1in.size(), 0);
    OCacheEntry cacheEntry = readBuffer.load(fileId, 0, false, writeBuffer, 1, true);
    readBuffer.release(cacheEntry, writeBuffer);
    Assert.assertEquals(am.size(), 0);
    Assert.assertEquals(a1out.size(), 0);
    Assert.assertEquals(a1in.size(), 1);
    readBuffer.clear();
    Assert.assertEquals(am.size(), 0);
    Assert.assertEquals(a1out.size(), 0);
    Assert.assertEquals(a1in.size(), 0);
    cacheEntry = readBuffer.load(fileId, 0, false, writeBuffer, 4, true);
    readBuffer.release(cacheEntry, writeBuffer);
    Assert.assertEquals(am.size(), 0);
    Assert.assertEquals(a1out.size(), 0);
    Assert.assertEquals(a1in.size(), 4);
}

Example 89 with OLogSequenceNumber

use of com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber in project orientdb by orientechnologies.

the class ODistributedAbstractPlugin method requestDatabaseFullSync.

protected boolean requestDatabaseFullSync(final ODistributedDatabaseImpl distrDatabase, final boolean backupDatabase, final String databaseName, final boolean iAskToAllNodes, final OModifiableDistributedConfiguration cfg) {
    // GET ALL THE OTHER SERVERS
    final Collection<String> nodes = cfg.getServers(null, nodeName);
    if (nodes.isEmpty()) {
        ODistributedServerLog.warn(this, nodeName, null, DIRECTION.NONE, "Cannot request full deploy of database '%s' because there are no nodes available with such database", databaseName);
        return false;
    }
    final List<String> selectedNodes = new ArrayList<String>();
    if (!iAskToAllNodes) {
        // VALID FOR FURTHER NODES
        for (String n : nodes) {
            if (isNodeStatusEqualsTo(n, databaseName, DB_STATUS.BACKUP)) {
                // SERVER ALREADY IN BACKUP: USE IT
                selectedNodes.add(n);
                break;
            }
        }
        if (selectedNodes.isEmpty()) {
            // GET THE FIRST ONE TO ASK FOR DATABASE. THIS FORCES TO HAVE ONE NODE TO DO BACKUP SAVING RESOURCES IN CASE BACKUP IS STILL
            // VALID FOR FURTHER NODES
            final Iterator<String> it = nodes.iterator();
            while (it.hasNext()) {
                final String f = it.next();
                if (isNodeStatusEqualsTo(f, databaseName, DB_STATUS.ONLINE, DB_STATUS.BACKUP)) {
                    selectedNodes.add(f);
                    break;
                }
            }
        }
    }
    if (selectedNodes.isEmpty())
        // NO NODE ONLINE, SEND THE MESSAGE TO EVERYONE
        selectedNodes.addAll(nodes);
    ODistributedServerLog.info(this, nodeName, selectedNodes.toString(), DIRECTION.OUT, "Requesting deploy of database '%s' on local server...", databaseName);
    final OLogSequenceNumber lastLSN = distrDatabase.getSyncConfiguration().getLastLSN(getLocalNodeName());
    final OAbstractReplicatedTask deployTask = new OSyncDatabaseTask(lastLSN, distrDatabase.getSyncConfiguration().getLastOperationTimestamp());
    final Map<String, Object> results = (Map<String, Object>) sendRequest(databaseName, null, selectedNodes, deployTask, getNextMessageIdCounter(), ODistributedRequest.EXECUTION_MODE.RESPONSE, null, null, null).getPayload();
    ODistributedServerLog.debug(this, nodeName, selectedNodes.toString(), DIRECTION.OUT, "Deploy returned: %s", results);
    final String dbPath = serverInstance.getDatabaseDirectory() + databaseName;
    // EXTRACT THE REAL RESULT
    for (Map.Entry<String, Object> r : results.entrySet()) {
        final Object value = r.getValue();
        if (value instanceof Boolean)
            continue;
        else if (value instanceof ODatabaseIsOldException) {
            // MANAGE THIS EXCEPTION AT UPPER LEVEL
            throw (ODatabaseIsOldException) value;
        } else if (value instanceof Throwable) {
            ODistributedServerLog.error(this, nodeName, r.getKey(), DIRECTION.IN, "Error on installing database '%s' in %s", (Exception) value, databaseName, dbPath);
            setDatabaseStatus(nodeName, databaseName, DB_STATUS.NOT_AVAILABLE);
            if (value instanceof ODistributedException)
                throw (ODistributedException) value;
        } else if (value instanceof ODistributedDatabaseChunk) {
            // DISABLED BECAUSE MOMENTUM IS NOT RELIABLE YET
            // distrDatabase.filterBeforeThisMomentum(((ODistributedDatabaseChunk) value).getMomentum());
            final File uniqueClustersBackupDirectory = getClusterOwnedExclusivelyByCurrentNode(dbPath, databaseName);
            // CLOSE THE STORAGE FIRST
            final ODistributedStorage stg = storages.remove(databaseName);
            if (stg != null)
                stg.close(true, false);
            if (backupDatabase)
                backupCurrentDatabase(databaseName);
            installDatabaseFromNetwork(dbPath, databaseName, distrDatabase, r.getKey(), (ODistributedDatabaseChunk) value, false, uniqueClustersBackupDirectory, cfg);
            OStorage storage = storages.get(databaseName);
            replaceStorageInSessions(storage);
            distrDatabase.resume();
            return true;
        } else
            throw new IllegalArgumentException("Type " + value + " not supported");
    }
    throw new ODistributedException("No response received from remote nodes for auto-deploy of database '" + databaseName + "'");
}

Example 90 with OLogSequenceNumber

use of com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber in project orientdb by orientechnologies.

the class ODistributedAbstractPlugin method requestDatabaseDelta.

public boolean requestDatabaseDelta(final ODistributedDatabaseImpl distrDatabase, final String databaseName, final OModifiableDistributedConfiguration cfg) {
    // GET ALL THE OTHER SERVERS
    final Collection<String> nodes = cfg.getServers(null, nodeName);
    getAvailableNodes(nodes, databaseName);
    if (nodes.size() == 0)
        return false;
    ODistributedServerLog.warn(this, nodeName, nodes.toString(), DIRECTION.OUT, "requesting delta database sync for '%s' on local server...", databaseName);
    // CREATE A MAP OF NODE/LSN BY READING LAST LSN SAVED
    final Map<String, OLogSequenceNumber> selectedNodes = new HashMap<String, OLogSequenceNumber>(nodes.size());
    for (String node : nodes) {
        final OLogSequenceNumber lsn = distrDatabase.getSyncConfiguration().getLastLSN(node);
        if (lsn != null) {
            selectedNodes.put(node, lsn);
        } else
            ODistributedServerLog.info(this, nodeName, node, DIRECTION.OUT, "Last LSN not found for database '%s', skip delta database sync", databaseName);
    }
    if (selectedNodes.isEmpty()) {
        // FORCE FULL DATABASE SYNC
        ODistributedServerLog.error(this, nodeName, null, DIRECTION.NONE, "No LSN found for delta sync for database '%s'. Asking for full database sync...", databaseName);
        throw new ODistributedDatabaseDeltaSyncException("Requested database delta sync but no LSN was found");
    }
    for (Map.Entry<String, OLogSequenceNumber> entry : selectedNodes.entrySet()) {
        final String targetNode = entry.getKey();
        final OLogSequenceNumber lsn = entry.getValue();
        final OSyncDatabaseDeltaTask deployTask = new OSyncDatabaseDeltaTask(lsn, distrDatabase.getSyncConfiguration().getLastOperationTimestamp());
        final List<String> targetNodes = new ArrayList<String>(1);
        targetNodes.add(targetNode);
        ODistributedServerLog.info(this, nodeName, targetNode, DIRECTION.OUT, "Requesting database delta sync for '%s' LSN=%s...", databaseName, lsn);
        try {
            final Map<String, Object> results = (Map<String, Object>) sendRequest(databaseName, null, targetNodes, deployTask, getNextMessageIdCounter(), ODistributedRequest.EXECUTION_MODE.RESPONSE, null, null, null).getPayload();
            ODistributedServerLog.debug(this, nodeName, selectedNodes.toString(), DIRECTION.OUT, "Database delta sync returned: %s", results);
            final String dbPath = serverInstance.getDatabaseDirectory() + databaseName;
            // EXTRACT THE REAL RESULT
            for (Map.Entry<String, Object> r : results.entrySet()) {
                final Object value = r.getValue();
                if (value instanceof Boolean)
                    continue;
                else {
                    final String server = r.getKey();
                    if (value instanceof ODistributedDatabaseDeltaSyncException) {
                        final ODistributedDatabaseDeltaSyncException exc = (ODistributedDatabaseDeltaSyncException) value;
                        ODistributedServerLog.warn(this, nodeName, server, DIRECTION.IN, "Error on installing database delta for '%s' (err=%s)", databaseName, exc.getMessage());
                        ODistributedServerLog.warn(this, nodeName, server, DIRECTION.IN, "Requesting full database '%s' sync...", databaseName);
                        // RESTORE STATUS TO ONLINE
                        setDatabaseStatus(server, databaseName, DB_STATUS.ONLINE);
                        throw (ODistributedDatabaseDeltaSyncException) value;
                    } else if (value instanceof ODatabaseIsOldException) {
                        // MANAGE THIS EXCEPTION AT UPPER LEVEL
                        throw (ODatabaseIsOldException) value;
                    } else if (value instanceof Throwable) {
                        ODistributedServerLog.error(this, nodeName, server, DIRECTION.IN, "Error on installing database delta %s in %s (%s)", value, databaseName, dbPath, value);
                        setDatabaseStatus(nodeName, databaseName, DB_STATUS.NOT_AVAILABLE);
                        return false;
                    } else if (value instanceof ODistributedDatabaseChunk) {
                        // distrDatabase.filterBeforeThisMomentum(((ODistributedDatabaseChunk) value).getMomentum());
                        // DISABLED BECAYSE THE MOMENTUM IS NOT YET RELIABLE
                        // distrDatabase.setParsing(true);
                        final File uniqueClustersBackupDirectory = getClusterOwnedExclusivelyByCurrentNode(dbPath, databaseName);
                        installDatabaseFromNetwork(dbPath, databaseName, distrDatabase, server, (ODistributedDatabaseChunk) value, true, uniqueClustersBackupDirectory, cfg);
                        ODistributedServerLog.info(this, nodeName, targetNode, DIRECTION.IN, "Installed delta of database '%s'...", databaseName);
                        if (!cfg.isSharded())
                            // DB NOT SHARDED, THE 1ST BACKUP IS GOOD
                            break;
                    } else
                        throw new IllegalArgumentException("Type " + value + " not supported");
                }
            }
        } catch (ODatabaseIsOldException e) {
            // FORWARD IT
            throw (ODatabaseIsOldException) e;
        } catch (Exception e) {
            ODistributedServerLog.error(this, nodeName, targetNode, DIRECTION.OUT, "Error on asking delta backup of database '%s' (err=%s)", databaseName, e.getMessage());
            throw new ODistributedDatabaseDeltaSyncException(lsn, e.toString());
        }
    }
    distrDatabase.resume();
    return true;
}