Examples with ODistributedDatabaseDeltaSyncException com.orientechnologies.orient.server.distributed.task.ODistributedDatabaseDeltaSyncException used on opensource projects

Example 1 with ODistributedDatabaseDeltaSyncException

use of com.orientechnologies.orient.server.distributed.task.ODistributedDatabaseDeltaSyncException in project orientdb by orientechnologies.

the class OSyncDatabaseDeltaTask method deltaBackup.

protected Object deltaBackup(final ODistributedRequestId requestId, final ODistributedServerManager iManager, final ODatabaseDocumentInternal database, final String databaseName) throws IOException, InterruptedException {
    final Long lastDeployment = (Long) iManager.getConfigurationMap().get(DEPLOYDB + databaseName);
    if (lastDeployment != null && lastDeployment.longValue() == random) {
        // SKIP IT
        ODistributedServerLog.debug(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.NONE, "Skip deploying delta database '%s' because already executed", databaseName);
        return Boolean.FALSE;
    }
    iManager.getConfigurationMap().put(DEPLOYDB + databaseName, random);
    final ODistributedDatabase dDatabase = checkIfCurrentDatabaseIsNotOlder(iManager, databaseName, startLSN);
    iManager.setDatabaseStatus(getNodeSource(), databaseName, ODistributedServerManager.DB_STATUS.SYNCHRONIZING);
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Deploying database '%s' with delta of changes...", databaseName);
    // CREATE A BACKUP OF DATABASE
    final File backupFile = new File(Orient.getTempPath() + "/backup_" + getNodeSource() + "_" + database.getName() + ".zip");
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Creating delta backup of database '%s' (startLSN=%s) in directory: %s...", databaseName, startLSN, backupFile.getAbsolutePath());
    if (backupFile.exists())
        backupFile.delete();
    else
        backupFile.getParentFile().mkdirs();
    backupFile.createNewFile();
    final FileOutputStream fileOutputStream = new FileOutputStream(backupFile);
    // final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream);
    final File completedFile = new File(backupFile.getAbsolutePath() + ".completed");
    if (completedFile.exists())
        completedFile.delete();
    final OStorage storage = database.getStorage().getUnderlying();
    if (!(storage instanceof OAbstractPaginatedStorage))
        throw new UnsupportedOperationException("Storage '" + storage.getName() + "' does not support distributed delta backup");
    final AtomicReference<OLogSequenceNumber> endLSN = new AtomicReference<OLogSequenceNumber>();
    final AtomicReference<ODistributedDatabaseDeltaSyncException> exception = new AtomicReference<ODistributedDatabaseDeltaSyncException>();
    try {
        final AtomicLong counter = new AtomicLong(0);
        endLSN.set(((OAbstractPaginatedStorage) storage).recordsChangedAfterLSN(startLSN, fileOutputStream, excludedClusterNames, new OCommandOutputListener() {

            @Override
            public void onMessage(final String iText) {
                if (iText.startsWith("read")) {
                    if (counter.incrementAndGet() % 100000 == 0) {
                        ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "- %s", iText);
                    }
                } else if (counter.incrementAndGet() % 10000 == 0) {
                    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "- %s", iText);
                }
            }
        }));
        if (endLSN.get() == null) {
            // DELTA NOT AVAILABLE, TRY WITH FULL BACKUP
            exception.set(new ODistributedDatabaseDeltaSyncException(startLSN));
        } else
            ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Delta backup of database '%s' completed. range=%s-%s", databaseName, startLSN, endLSN.get());
    } catch (Exception e) {
        // UNKNOWN ERROR, DELTA NOT AVAILABLE, TRY WITH FULL BACKUP
        exception.set(new ODistributedDatabaseDeltaSyncException(startLSN, e.getMessage()));
    } finally {
        try {
            fileOutputStream.close();
        } catch (IOException e) {
        }
        try {
            completedFile.createNewFile();
        } catch (IOException e) {
            OLogManager.instance().error(this, "Cannot create file of delta backup completed: %s", e, completedFile);
        }
    }
    if (exception.get() instanceof ODistributedDatabaseDeltaSyncException) {
        throw exception.get();
    }
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Deploy delta database task completed");
    // GET THE MOMENTUM, BUT OVERWRITE THE LAST LSN RECEIVED FROM THE DELTA
    final ODistributedMomentum momentum = dDatabase.getSyncConfiguration().getMomentum().copy();
    momentum.setLSN(iManager.getLocalNodeName(), endLSN.get());
    final ODistributedDatabaseChunk chunk = new ODistributedDatabaseChunk(backupFile, 0, CHUNK_MAX_SIZE, momentum, false);
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "- transferring chunk #%d offset=%d size=%s...", 1, 0, OFileUtils.getSizeAsNumber(chunk.buffer.length));
    if (chunk.last)
        // NO MORE CHUNKS: SET THE NODE ONLINE (SYNCHRONIZING ENDED)
        iManager.setDatabaseStatus(iManager.getLocalNodeName(), databaseName, ODistributedServerManager.DB_STATUS.ONLINE);
    return chunk;
}

Example 2 with ODistributedDatabaseDeltaSyncException

use of com.orientechnologies.orient.server.distributed.task.ODistributedDatabaseDeltaSyncException in project orientdb by orientechnologies.

the class OIncrementalServerSync method importDelta.

/**
   * Deleted records are written in output stream first, then created/updated records. All records are sorted by record id.
   * <p>
   * Each record in output stream is written using following format:
   * <ol>
   * <li>Record's cluster id - 4 bytes</li>
   * <li>Record's cluster position - 8 bytes</li>
   * <li>Delete flag, 1 if record is deleted - 1 byte</li>
   * <li>Record version , only if record is not deleted - 4 bytes</li>
   * <li>Record type, only if record is not deleted - 1 byte</li>
   * <li>Length of binary presentation of record, only if record is not deleted - 4 bytes</li>
   * <li>Binary presentation of the record, only if record is not deleted - length of content is provided in above entity</li>
   * </ol>
   */
public void importDelta(final OServer serverInstance, final ODatabaseDocumentInternal db, final FileInputStream in, final String iNode) throws IOException {
    final String nodeName = serverInstance.getDistributedManager().getLocalNodeName();
    try {
        serverInstance.openDatabase(db);
        OScenarioThreadLocal.executeAsDistributed(new Callable<Object>() {

            @Override
            public Object call() throws Exception {
                db.activateOnCurrentThread();
                long totalRecords = 0;
                long totalCreated = 0;
                long totalUpdated = 0;
                long totalDeleted = 0;
                long totalHoles = 0;
                long totalSkipped = 0;
                ODistributedServerLog.info(this, nodeName, iNode, DIRECTION.IN, "Started import of delta for database '" + db.getName() + "'");
                long lastLap = System.currentTimeMillis();
                // final GZIPInputStream gzipInput = new GZIPInputStream(in);
                try {
                    final DataInputStream input = new DataInputStream(in);
                    try {
                        final long records = input.readLong();
                        for (long i = 0; i < records; ++i) {
                            final int clusterId = input.readInt();
                            final long clusterPos = input.readLong();
                            final boolean deleted = input.readBoolean();
                            final ORecordId rid = new ORecordId(clusterId, clusterPos);
                            totalRecords++;
                            final OPaginatedCluster cluster = (OPaginatedCluster) db.getStorage().getUnderlying().getClusterById(rid.getClusterId());
                            final OPaginatedCluster.RECORD_STATUS recordStatus = cluster.getRecordStatus(rid.getClusterPosition());
                            ORecord newRecord = null;
                            if (deleted) {
                                ODistributedServerLog.debug(this, nodeName, iNode, DIRECTION.IN, "DELTA <- deleting %s", rid);
                                switch(recordStatus) {
                                    case REMOVED:
                                        // SKIP IT
                                        totalSkipped++;
                                        continue;
                                    case ALLOCATED:
                                    case PRESENT:
                                        // DELETE IT
                                        db.delete(rid);
                                        break;
                                    case NOT_EXISTENT:
                                        totalSkipped++;
                                        break;
                                }
                                totalDeleted++;
                            } else {
                                final int recordVersion = input.readInt();
                                final int recordType = input.readByte();
                                final int recordSize = input.readInt();
                                final byte[] recordContent = new byte[recordSize];
                                input.read(recordContent);
                                switch(recordStatus) {
                                    case REMOVED:
                                        // SKIP IT
                                        totalSkipped++;
                                        continue;
                                    case ALLOCATED:
                                    case PRESENT:
                                        // UPDATE IT
                                        newRecord = Orient.instance().getRecordFactoryManager().newInstance((byte) recordType);
                                        ORecordInternal.fill(newRecord, rid, ORecordVersionHelper.setRollbackMode(recordVersion), recordContent, true);
                                        final ORecord loadedRecord = rid.getRecord();
                                        if (loadedRecord instanceof ODocument) {
                                            // APPLY CHANGES FIELD BY FIELD TO MARK DIRTY FIELDS FOR INDEXES/HOOKS
                                            ODocument loadedDocument = (ODocument) loadedRecord;
                                            loadedDocument.merge((ODocument) newRecord, false, false);
                                            ORecordInternal.setVersion(loadedRecord, ORecordVersionHelper.setRollbackMode(recordVersion));
                                            loadedDocument.setDirty();
                                            newRecord = loadedDocument;
                                        }
                                        // SAVE THE UPDATE RECORD
                                        newRecord.save();
                                        ODistributedServerLog.debug(this, nodeName, iNode, DIRECTION.IN, "DELTA <- updating rid=%s type=%d size=%d v=%d content=%s", rid, recordType, recordSize, recordVersion, newRecord);
                                        totalUpdated++;
                                        break;
                                    case NOT_EXISTENT:
                                        // CREATE AND DELETE RECORD IF NEEDED
                                        do {
                                            newRecord = Orient.instance().getRecordFactoryManager().newInstance((byte) recordType);
                                            ORecordInternal.fill(newRecord, new ORecordId(rid.getClusterId(), -1), recordVersion - 1, recordContent, true);
                                            try {
                                                newRecord.save();
                                            } catch (ORecordNotFoundException e) {
                                                ODistributedServerLog.info(this, nodeName, iNode, DIRECTION.IN, "DELTA <- error on saving record (not found) rid=%s type=%d size=%d v=%d content=%s", rid, recordType, recordSize, recordVersion, newRecord);
                                            } catch (ORecordDuplicatedException e) {
                                                ODistributedServerLog.info(this, nodeName, iNode, DIRECTION.IN, "DELTA <- error on saving record (duplicated %s) rid=%s type=%d size=%d v=%d content=%s", e.getRid(), rid, recordType, recordSize, recordVersion, newRecord);
                                                // throw OException.wrapException(
                                                // new ODistributedDatabaseDeltaSyncException("Error on delta sync: found duplicated record " + rid), e);
                                                final ORecord duplicatedRecord = db.load(e.getRid(), null, true);
                                                if (duplicatedRecord == null) {
                                                    // RECORD REMOVED: THE INDEX IS DIRTY, FIX THE DIRTY INDEX
                                                    final ODocument doc = (ODocument) newRecord;
                                                    final OIndex<?> index = db.getMetadata().getIndexManager().getIndex(e.getIndexName());
                                                    final List<String> fields = index.getDefinition().getFields();
                                                    final List<Object> values = new ArrayList<Object>(fields.size());
                                                    for (String f : fields) {
                                                        values.add(doc.field(f));
                                                    }
                                                    final Object keyValue = index.getDefinition().createValue(values);
                                                    index.remove(keyValue, e.getRid());
                                                    // RESAVE THE RECORD
                                                    newRecord.save();
                                                } else
                                                    break;
                                            }
                                            if (newRecord.getIdentity().getClusterPosition() < clusterPos) {
                                                // DELETE THE RECORD TO CREATE A HOLE
                                                ODistributedServerLog.debug(this, nodeName, iNode, DIRECTION.IN, "DELTA <- creating hole rid=%s", newRecord.getIdentity());
                                                newRecord.delete();
                                                totalHoles++;
                                            }
                                        } while (newRecord.getIdentity().getClusterPosition() < clusterPos);
                                        ODistributedServerLog.debug(this, nodeName, iNode, DIRECTION.IN, "DELTA <- creating rid=%s type=%d size=%d v=%d content=%s", rid, recordType, recordSize, recordVersion, newRecord);
                                        totalCreated++;
                                        break;
                                }
                                if (newRecord.getIdentity().isPersistent() && !newRecord.getIdentity().equals(rid))
                                    throw new ODistributedDatabaseDeltaSyncException("Error on synchronization of records, rids are different: saved " + newRecord.getIdentity() + ", but it should be " + rid);
                            }
                            final long now = System.currentTimeMillis();
                            if (now - lastLap > 2000) {
                                // DUMP STATS EVERY SECOND
                                ODistributedServerLog.info(this, nodeName, iNode, DIRECTION.IN, "- %,d total entries: %,d created, %,d updated, %,d deleted, %,d holes, %,d skipped...", totalRecords, totalCreated, totalUpdated, totalDeleted, totalHoles, totalSkipped);
                                lastLap = now;
                            }
                        }
                        db.getMetadata().reload();
                    } finally {
                        input.close();
                    }
                } catch (Exception e) {
                    ODistributedServerLog.error(this, nodeName, iNode, DIRECTION.IN, "Error on installing database delta '%s' on local server", e, db.getName());
                    throw OException.wrapException(new ODistributedException("Error on installing database delta '" + db.getName() + "' on local server"), e);
                } finally {
                // gzipInput.close();
                }
                ODistributedServerLog.info(this, nodeName, iNode, DIRECTION.IN, "Installed database delta for '%s'. %d total entries: %d created, %d updated, %d deleted, %d holes, %,d skipped", db.getName(), totalRecords, totalCreated, totalUpdated, totalDeleted, totalHoles, totalSkipped);
                return null;
            }
        });
        db.activateOnCurrentThread();
    } catch (Exception e) {
        // FORCE FULL DATABASE SYNC
        ODistributedServerLog.error(this, nodeName, iNode, DIRECTION.IN, "Error while applying changes of database delta sync on '%s': forcing full database sync...", e, db.getName());
        throw OException.wrapException(new ODistributedDatabaseDeltaSyncException("Error while applying changes of database delta sync on '" + db.getName() + "': forcing full database sync..."), e);
    }
}

Example 3 with ODistributedDatabaseDeltaSyncException

use of com.orientechnologies.orient.server.distributed.task.ODistributedDatabaseDeltaSyncException in project orientdb by orientechnologies.

the class ODistributedAbstractPlugin method requestDatabaseDelta.

public boolean requestDatabaseDelta(final ODistributedDatabaseImpl distrDatabase, final String databaseName, final OModifiableDistributedConfiguration cfg) {
    // GET ALL THE OTHER SERVERS
    final Collection<String> nodes = cfg.getServers(null, nodeName);
    getAvailableNodes(nodes, databaseName);
    if (nodes.size() == 0)
        return false;
    ODistributedServerLog.warn(this, nodeName, nodes.toString(), DIRECTION.OUT, "requesting delta database sync for '%s' on local server...", databaseName);
    // CREATE A MAP OF NODE/LSN BY READING LAST LSN SAVED
    final Map<String, OLogSequenceNumber> selectedNodes = new HashMap<String, OLogSequenceNumber>(nodes.size());
    for (String node : nodes) {
        final OLogSequenceNumber lsn = distrDatabase.getSyncConfiguration().getLastLSN(node);
        if (lsn != null) {
            selectedNodes.put(node, lsn);
        } else
            ODistributedServerLog.info(this, nodeName, node, DIRECTION.OUT, "Last LSN not found for database '%s', skip delta database sync", databaseName);
    }
    if (selectedNodes.isEmpty()) {
        // FORCE FULL DATABASE SYNC
        ODistributedServerLog.error(this, nodeName, null, DIRECTION.NONE, "No LSN found for delta sync for database '%s'. Asking for full database sync...", databaseName);
        throw new ODistributedDatabaseDeltaSyncException("Requested database delta sync but no LSN was found");
    }
    for (Map.Entry<String, OLogSequenceNumber> entry : selectedNodes.entrySet()) {
        final String targetNode = entry.getKey();
        final OLogSequenceNumber lsn = entry.getValue();
        final OSyncDatabaseDeltaTask deployTask = new OSyncDatabaseDeltaTask(lsn, distrDatabase.getSyncConfiguration().getLastOperationTimestamp());
        final List<String> targetNodes = new ArrayList<String>(1);
        targetNodes.add(targetNode);
        ODistributedServerLog.info(this, nodeName, targetNode, DIRECTION.OUT, "Requesting database delta sync for '%s' LSN=%s...", databaseName, lsn);
        try {
            final Map<String, Object> results = (Map<String, Object>) sendRequest(databaseName, null, targetNodes, deployTask, getNextMessageIdCounter(), ODistributedRequest.EXECUTION_MODE.RESPONSE, null, null).getPayload();
            ODistributedServerLog.debug(this, nodeName, selectedNodes.toString(), DIRECTION.OUT, "Database delta sync returned: %s", results);
            final String dbPath = serverInstance.getDatabaseDirectory() + databaseName;
            // EXTRACT THE REAL RESULT
            for (Map.Entry<String, Object> r : results.entrySet()) {
                final Object value = r.getValue();
                if (value instanceof Boolean)
                    continue;
                else {
                    final String server = r.getKey();
                    if (value instanceof ODistributedDatabaseDeltaSyncException) {
                        final ODistributedDatabaseDeltaSyncException exc = (ODistributedDatabaseDeltaSyncException) value;
                        ODistributedServerLog.warn(this, nodeName, server, DIRECTION.IN, "Error on installing database delta for '%s' (err=%s)", databaseName, exc.getMessage());
                        ODistributedServerLog.warn(this, nodeName, server, DIRECTION.IN, "Requesting full database '%s' sync...", databaseName);
                        // RESTORE STATUS TO ONLINE
                        setDatabaseStatus(server, databaseName, DB_STATUS.ONLINE);
                        throw (ODistributedDatabaseDeltaSyncException) value;
                    } else if (value instanceof ODatabaseIsOldException) {
                        // MANAGE THIS EXCEPTION AT UPPER LEVEL
                        throw (ODatabaseIsOldException) value;
                    } else if (value instanceof Throwable) {
                        ODistributedServerLog.error(this, nodeName, server, DIRECTION.IN, "Error on installing database delta %s in %s (%s)", value, databaseName, dbPath, value);
                        setDatabaseStatus(nodeName, databaseName, DB_STATUS.NOT_AVAILABLE);
                        return false;
                    } else if (value instanceof ODistributedDatabaseChunk) {
                        // distrDatabase.filterBeforeThisMomentum(((ODistributedDatabaseChunk) value).getMomentum());
                        // DISABLED BECAYSE THE MOMENTUM IS NOT YET RELIABLE
                        // distrDatabase.setParsing(true);
                        final File uniqueClustersBackupDirectory = getClusterOwnedExclusivelyByCurrentNode(dbPath, databaseName);
                        installDatabaseFromNetwork(dbPath, databaseName, distrDatabase, server, (ODistributedDatabaseChunk) value, true, uniqueClustersBackupDirectory, cfg);
                        ODistributedServerLog.info(this, nodeName, targetNode, DIRECTION.IN, "Installed delta of database '%s'...", databaseName);
                        if (!cfg.isSharded())
                            // DB NOT SHARDED, THE 1ST BACKUP IS GOOD
                            break;
                    } else
                        throw new IllegalArgumentException("Type " + value + " not supported");
                }
            }
        } catch (ODatabaseIsOldException e) {
            // FORWARD IT
            throw (ODatabaseIsOldException) e;
        } catch (Exception e) {
            ODistributedServerLog.error(this, nodeName, targetNode, DIRECTION.OUT, "Error on asking delta backup of database '%s' (err=%s)", databaseName, e.getMessage());
            throw new ODistributedDatabaseDeltaSyncException(lsn, e.toString());
        }
    }
    distrDatabase.resume();
    return true;
}