Examples with OLogSequenceNumber com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber used on opensource projects

Example 6 with OLogSequenceNumber

use of com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber in project orientdb by orientechnologies.

the class OSyncDatabaseDeltaTask method deltaBackup.

protected Object deltaBackup(final ODistributedRequestId requestId, final ODistributedServerManager iManager, final ODatabaseDocumentInternal database, final String databaseName) throws IOException, InterruptedException {
    final Long lastDeployment = (Long) iManager.getConfigurationMap().get(DEPLOYDB + databaseName);
    if (lastDeployment != null && lastDeployment.longValue() == random) {
        // SKIP IT
        ODistributedServerLog.debug(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.NONE, "Skip deploying delta database '%s' because already executed", databaseName);
        return Boolean.FALSE;
    }
    iManager.getConfigurationMap().put(DEPLOYDB + databaseName, random);
    final ODistributedDatabase dDatabase = checkIfCurrentDatabaseIsNotOlder(iManager, databaseName, startLSN);
    iManager.setDatabaseStatus(getNodeSource(), databaseName, ODistributedServerManager.DB_STATUS.SYNCHRONIZING);
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Deploying database '%s' with delta of changes...", databaseName);
    // CREATE A BACKUP OF DATABASE
    final File backupFile = new File(Orient.getTempPath() + "/backup_" + getNodeSource() + "_" + database.getName() + ".zip");
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Creating delta backup of database '%s' (startLSN=%s) in directory: %s...", databaseName, startLSN, backupFile.getAbsolutePath());
    if (backupFile.exists())
        backupFile.delete();
    else
        backupFile.getParentFile().mkdirs();
    backupFile.createNewFile();
    final FileOutputStream fileOutputStream = new FileOutputStream(backupFile);
    // final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream);
    final File completedFile = new File(backupFile.getAbsolutePath() + ".completed");
    if (completedFile.exists())
        completedFile.delete();
    final OStorage storage = database.getStorage().getUnderlying();
    if (!(storage instanceof OAbstractPaginatedStorage))
        throw new UnsupportedOperationException("Storage '" + storage.getName() + "' does not support distributed delta backup");
    final AtomicReference<OLogSequenceNumber> endLSN = new AtomicReference<OLogSequenceNumber>();
    final AtomicReference<ODistributedDatabaseDeltaSyncException> exception = new AtomicReference<ODistributedDatabaseDeltaSyncException>();
    try {
        final AtomicLong counter = new AtomicLong(0);
        endLSN.set(((OAbstractPaginatedStorage) storage).recordsChangedAfterLSN(startLSN, fileOutputStream, excludedClusterNames, new OCommandOutputListener() {

            @Override
            public void onMessage(final String iText) {
                if (iText.startsWith("read")) {
                    if (counter.incrementAndGet() % 100000 == 0) {
                        ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "- %s", iText);
                    }
                } else if (counter.incrementAndGet() % 10000 == 0) {
                    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "- %s", iText);
                }
            }
        }));
        if (endLSN.get() == null) {
            // DELTA NOT AVAILABLE, TRY WITH FULL BACKUP
            exception.set(new ODistributedDatabaseDeltaSyncException(startLSN));
        } else
            ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Delta backup of database '%s' completed. range=%s-%s", databaseName, startLSN, endLSN.get());
    } catch (Exception e) {
        // UNKNOWN ERROR, DELTA NOT AVAILABLE, TRY WITH FULL BACKUP
        exception.set(new ODistributedDatabaseDeltaSyncException(startLSN, e.getMessage()));
    } finally {
        try {
            fileOutputStream.close();
        } catch (IOException e) {
        }
        try {
            completedFile.createNewFile();
        } catch (IOException e) {
            OLogManager.instance().error(this, "Cannot create file of delta backup completed: %s", e, completedFile);
        }
    }
    if (exception.get() instanceof ODistributedDatabaseDeltaSyncException) {
        throw exception.get();
    }
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Deploy delta database task completed");
    // GET THE MOMENTUM, BUT OVERWRITE THE LAST LSN RECEIVED FROM THE DELTA
    final ODistributedMomentum momentum = dDatabase.getSyncConfiguration().getMomentum().copy();
    momentum.setLSN(iManager.getLocalNodeName(), endLSN.get());
    final ODistributedDatabaseChunk chunk = new ODistributedDatabaseChunk(backupFile, 0, CHUNK_MAX_SIZE, momentum, false);
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "- transferring chunk #%d offset=%d size=%s...", 1, 0, OFileUtils.getSizeAsNumber(chunk.buffer.length));
    if (chunk.last)
        // NO MORE CHUNKS: SET THE NODE ONLINE (SYNCHRONIZING ENDED)
        iManager.setDatabaseStatus(iManager.getLocalNodeName(), databaseName, ODistributedServerManager.DB_STATUS.ONLINE);
    return chunk;
}

Example 7 with OLogSequenceNumber

use of com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber in project orientdb by orientechnologies.

the class OSyncDatabaseDeltaTask method fromStream.

@Override
public void fromStream(final DataInput in, final ORemoteTaskFactory factory) throws IOException {
    startLSN = new OLogSequenceNumber(in);
    lastOperationTimestamp = in.readLong();
    random = in.readLong();
    excludedClusterNames.clear();
    final int total = in.readInt();
    for (int i = 0; i < total; ++i) {
        excludedClusterNames.add(in.readUTF());
    }
}

Example 8 with OLogSequenceNumber

use of com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber in project orientdb by orientechnologies.

the class OSBTreeBonsaiNonLeafBucketTest method testSearch.

public void testSearch() throws Exception {
    long seed = System.currentTimeMillis();
    System.out.println("testSearch seed : " + seed);
    TreeSet<Long> keys = new TreeSet<Long>();
    Random random = new Random(seed);
    while (keys.size() < 2 * OSBTreeBonsaiBucket.MAX_BUCKET_SIZE_BYTES / OLongSerializer.LONG_SIZE) {
        keys.add(random.nextLong());
    }
    OByteBufferPool bufferPool = OByteBufferPool.instance();
    ByteBuffer buffer = bufferPool.acquireDirect(true);
    OCachePointer cachePointer = new OCachePointer(buffer, bufferPool, new OLogSequenceNumber(0, 0), 0, 0);
    cachePointer.incrementReferrer();
    OCacheEntry cacheEntry = new OCacheEntry(0, 0, cachePointer, false);
    cacheEntry.acquireExclusiveLock();
    OSBTreeBonsaiBucket<Long, OIdentifiable> treeBucket = new OSBTreeBonsaiBucket<Long, OIdentifiable>(cacheEntry, 0, false, OLongSerializer.INSTANCE, OLinkSerializer.INSTANCE, null, null);
    int index = 0;
    Map<Long, Integer> keyIndexMap = new HashMap<Long, Integer>();
    for (Long key : keys) {
        if (!treeBucket.addEntry(index, new OSBTreeBonsaiBucket.SBTreeEntry<Long, OIdentifiable>(new OBonsaiBucketPointer(random.nextInt(Integer.MAX_VALUE), 8192 * 2), new OBonsaiBucketPointer(random.nextInt(Integer.MAX_VALUE), 8192 * 2), key, null), true))
            break;
        keyIndexMap.put(key, index);
        index++;
    }
    Assert.assertEquals(treeBucket.size(), keyIndexMap.size());
    for (Map.Entry<Long, Integer> keyIndexEntry : keyIndexMap.entrySet()) {
        int bucketIndex = treeBucket.find(keyIndexEntry.getKey());
        Assert.assertEquals(bucketIndex, (int) keyIndexEntry.getValue());
    }
    OBonsaiBucketPointer prevRight = OBonsaiBucketPointer.NULL;
    for (int i = 0; i < treeBucket.size(); i++) {
        OSBTreeBonsaiBucket.SBTreeEntry<Long, OIdentifiable> entry = treeBucket.getEntry(i);
        if (prevRight.getPageIndex() > 0)
            Assert.assertEquals(entry.leftChild, prevRight);
        prevRight = entry.rightChild;
    }
    OBonsaiBucketPointer prevLeft = OBonsaiBucketPointer.NULL;
    for (int i = treeBucket.size() - 1; i >= 0; i--) {
        OSBTreeBonsaiBucket.SBTreeEntry<Long, OIdentifiable> entry = treeBucket.getEntry(i);
        if (prevLeft.getPageIndex() > 0)
            Assert.assertEquals(entry.rightChild, prevLeft);
        prevLeft = entry.leftChild;
    }
    cacheEntry.releaseExclusiveLock();
    cachePointer.decrementReferrer();
}

Example 9 with OLogSequenceNumber

use of com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber in project orientdb by orientechnologies.

the class OSBTreeBonsaiNonLeafBucketTest method testShrink.

public void testShrink() throws Exception {
    long seed = System.currentTimeMillis();
    System.out.println("testShrink seed : " + seed);
    TreeSet<Long> keys = new TreeSet<Long>();
    Random random = new Random(seed);
    while (keys.size() < 2 * OSBTreeBonsaiBucket.MAX_BUCKET_SIZE_BYTES / OLongSerializer.LONG_SIZE) {
        keys.add(random.nextLong());
    }
    OByteBufferPool bufferPool = OByteBufferPool.instance();
    ByteBuffer buffer = bufferPool.acquireDirect(true);
    OCachePointer cachePointer = new OCachePointer(buffer, bufferPool, new OLogSequenceNumber(0, 0), 0, 0);
    cachePointer.incrementReferrer();
    OCacheEntry cacheEntry = new OCacheEntry(0, 0, cachePointer, false);
    cacheEntry.acquireExclusiveLock();
    OSBTreeBonsaiBucket<Long, OIdentifiable> treeBucket = new OSBTreeBonsaiBucket<Long, OIdentifiable>(cacheEntry, 0, false, OLongSerializer.INSTANCE, OLinkSerializer.INSTANCE, null, null);
    int index = 0;
    for (Long key : keys) {
        if (!treeBucket.addEntry(index, new OSBTreeBonsaiBucket.SBTreeEntry<Long, OIdentifiable>(new OBonsaiBucketPointer(index, 8192 * 2), new OBonsaiBucketPointer(index + 1, 8192 * 2), key, null), true))
            break;
        index++;
    }
    int originalSize = treeBucket.size();
    treeBucket.shrink(treeBucket.size() / 2);
    Assert.assertEquals(treeBucket.size(), index / 2);
    index = 0;
    final Map<Long, Integer> keyIndexMap = new HashMap<Long, Integer>();
    Iterator<Long> keysIterator = keys.iterator();
    while (keysIterator.hasNext() && index < treeBucket.size()) {
        Long key = keysIterator.next();
        keyIndexMap.put(key, index);
        index++;
    }
    for (Map.Entry<Long, Integer> keyIndexEntry : keyIndexMap.entrySet()) {
        int bucketIndex = treeBucket.find(keyIndexEntry.getKey());
        Assert.assertEquals(bucketIndex, (int) keyIndexEntry.getValue());
    }
    for (Map.Entry<Long, Integer> keyIndexEntry : keyIndexMap.entrySet()) {
        OSBTreeBonsaiBucket.SBTreeEntry<Long, OIdentifiable> entry = treeBucket.getEntry(keyIndexEntry.getValue());
        Assert.assertEquals(entry, new OSBTreeBonsaiBucket.SBTreeEntry<Long, OIdentifiable>(new OBonsaiBucketPointer(keyIndexEntry.getValue(), 8192 * 2), new OBonsaiBucketPointer(keyIndexEntry.getValue() + 1, 8192 * 2), keyIndexEntry.getKey(), null));
    }
    int keysToAdd = originalSize - treeBucket.size();
    int addedKeys = 0;
    while (keysIterator.hasNext() && index < originalSize) {
        Long key = keysIterator.next();
        if (!treeBucket.addEntry(index, new OSBTreeBonsaiBucket.SBTreeEntry<Long, OIdentifiable>(new OBonsaiBucketPointer(index, 8192 * 2), new OBonsaiBucketPointer(index + 1, 8192 * 2), key, null), true))
            break;
        keyIndexMap.put(key, index);
        index++;
        addedKeys++;
    }
    for (Map.Entry<Long, Integer> keyIndexEntry : keyIndexMap.entrySet()) {
        OSBTreeBonsaiBucket.SBTreeEntry<Long, OIdentifiable> entry = treeBucket.getEntry(keyIndexEntry.getValue());
        Assert.assertEquals(entry, new OSBTreeBonsaiBucket.SBTreeEntry<Long, OIdentifiable>(new OBonsaiBucketPointer(keyIndexEntry.getValue(), 8192 * 2), new OBonsaiBucketPointer(keyIndexEntry.getValue() + 1, 8192 * 2), keyIndexEntry.getKey(), null));
    }
    Assert.assertEquals(treeBucket.size(), originalSize);
    Assert.assertEquals(addedKeys, keysToAdd);
    cacheEntry.releaseExclusiveLock();
    cachePointer.decrementReferrer();
}

Example 10 with OLogSequenceNumber

use of com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber in project orientdb by orientechnologies.

the class OSBTreeBonsaiLeafBucketTest method testSetLeftSibling.

public void testSetLeftSibling() throws Exception {
    OByteBufferPool bufferPool = OByteBufferPool.instance();
    ByteBuffer buffer = bufferPool.acquireDirect(true);
    OCachePointer cachePointer = new OCachePointer(buffer, bufferPool, new OLogSequenceNumber(0, 0), 0, 0);
    cachePointer.incrementReferrer();
    OCacheEntry cacheEntry = new OCacheEntry(0, 0, cachePointer, false);
    cacheEntry.acquireExclusiveLock();
    OSBTreeBonsaiBucket<Long, OIdentifiable> treeBucket = new OSBTreeBonsaiBucket<Long, OIdentifiable>(cacheEntry, 0, true, OLongSerializer.INSTANCE, OLinkSerializer.INSTANCE, null, null);
    final OBonsaiBucketPointer p = new OBonsaiBucketPointer(123, 8192 * 2);
    treeBucket.setLeftSibling(p);
    Assert.assertEquals(treeBucket.getLeftSibling(), p);
    cacheEntry.releaseExclusiveLock();
    cachePointer.decrementReferrer();
}