Examples with OCommandOutputListener com.orientechnologies.orient.core.command.OCommandOutputListener used on opensource projects

Example 1 with OCommandOutputListener

use of com.orientechnologies.orient.core.command.OCommandOutputListener in project orientdb by orientechnologies.

the class OSyncDatabaseDeltaTask method deltaBackup.

protected Object deltaBackup(final ODistributedRequestId requestId, final ODistributedServerManager iManager, final ODatabaseDocumentInternal database, final String databaseName) throws IOException, InterruptedException {
    final Long lastDeployment = (Long) iManager.getConfigurationMap().get(DEPLOYDB + databaseName);
    if (lastDeployment != null && lastDeployment.longValue() == random) {
        // SKIP IT
        ODistributedServerLog.debug(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.NONE, "Skip deploying delta database '%s' because already executed", databaseName);
        return Boolean.FALSE;
    }
    iManager.getConfigurationMap().put(DEPLOYDB + databaseName, random);
    final ODistributedDatabase dDatabase = checkIfCurrentDatabaseIsNotOlder(iManager, databaseName, startLSN);
    iManager.setDatabaseStatus(getNodeSource(), databaseName, ODistributedServerManager.DB_STATUS.SYNCHRONIZING);
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Deploying database '%s' with delta of changes...", databaseName);
    // CREATE A BACKUP OF DATABASE
    final File backupFile = new File(Orient.getTempPath() + "/backup_" + getNodeSource() + "_" + database.getName() + ".zip");
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Creating delta backup of database '%s' (startLSN=%s) in directory: %s...", databaseName, startLSN, backupFile.getAbsolutePath());
    if (backupFile.exists())
        backupFile.delete();
    else
        backupFile.getParentFile().mkdirs();
    backupFile.createNewFile();
    final FileOutputStream fileOutputStream = new FileOutputStream(backupFile);
    // final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream);
    final File completedFile = new File(backupFile.getAbsolutePath() + ".completed");
    if (completedFile.exists())
        completedFile.delete();
    final OStorage storage = database.getStorage().getUnderlying();
    if (!(storage instanceof OAbstractPaginatedStorage))
        throw new UnsupportedOperationException("Storage '" + storage.getName() + "' does not support distributed delta backup");
    final AtomicReference<OLogSequenceNumber> endLSN = new AtomicReference<OLogSequenceNumber>();
    final AtomicReference<ODistributedDatabaseDeltaSyncException> exception = new AtomicReference<ODistributedDatabaseDeltaSyncException>();
    try {
        final AtomicLong counter = new AtomicLong(0);
        endLSN.set(((OAbstractPaginatedStorage) storage).recordsChangedAfterLSN(startLSN, fileOutputStream, excludedClusterNames, new OCommandOutputListener() {

            @Override
            public void onMessage(final String iText) {
                if (iText.startsWith("read")) {
                    if (counter.incrementAndGet() % 100000 == 0) {
                        ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "- %s", iText);
                    }
                } else if (counter.incrementAndGet() % 10000 == 0) {
                    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "- %s", iText);
                }
            }
        }));
        if (endLSN.get() == null) {
            // DELTA NOT AVAILABLE, TRY WITH FULL BACKUP
            exception.set(new ODistributedDatabaseDeltaSyncException(startLSN));
        } else
            ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Delta backup of database '%s' completed. range=%s-%s", databaseName, startLSN, endLSN.get());
    } catch (Exception e) {
        // UNKNOWN ERROR, DELTA NOT AVAILABLE, TRY WITH FULL BACKUP
        exception.set(new ODistributedDatabaseDeltaSyncException(startLSN, e.getMessage()));
    } finally {
        try {
            fileOutputStream.close();
        } catch (IOException e) {
        }
        try {
            completedFile.createNewFile();
        } catch (IOException e) {
            OLogManager.instance().error(this, "Cannot create file of delta backup completed: %s", e, completedFile);
        }
    }
    if (exception.get() instanceof ODistributedDatabaseDeltaSyncException) {
        throw exception.get();
    }
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Deploy delta database task completed");
    // GET THE MOMENTUM, BUT OVERWRITE THE LAST LSN RECEIVED FROM THE DELTA
    final ODistributedMomentum momentum = dDatabase.getSyncConfiguration().getMomentum().copy();
    momentum.setLSN(iManager.getLocalNodeName(), endLSN.get());
    final ODistributedDatabaseChunk chunk = new ODistributedDatabaseChunk(backupFile, 0, CHUNK_MAX_SIZE, momentum, false);
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "- transferring chunk #%d offset=%d size=%s...", 1, 0, OFileUtils.getSizeAsNumber(chunk.buffer.length));
    if (chunk.last)
        // NO MORE CHUNKS: SET THE NODE ONLINE (SYNCHRONIZING ENDED)
        iManager.setDatabaseStatus(iManager.getLocalNodeName(), databaseName, ODistributedServerManager.DB_STATUS.ONLINE);
    return chunk;
}

Example 2 with OCommandOutputListener

use of com.orientechnologies.orient.core.command.OCommandOutputListener in project orientdb by orientechnologies.

the class OSyncDatabaseTask method execute.

@Override
public Object execute(final ODistributedRequestId requestId, final OServer iServer, final ODistributedServerManager iManager, final ODatabaseDocumentInternal database) throws Exception {
    if (!iManager.getLocalNodeName().equals(getNodeSource())) {
        if (database == null)
            throw new ODistributedException("Database instance is null");
        final String databaseName = database.getName();
        final ODistributedDatabase dDatabase = checkIfCurrentDatabaseIsNotOlder(iManager, databaseName, null);
        try {
            final Long lastDeployment = (Long) iManager.getConfigurationMap().get(DEPLOYDB + databaseName);
            if (lastDeployment != null && lastDeployment.longValue() == random) {
                // SKIP IT
                ODistributedServerLog.debug(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.NONE, "Skip deploying database '%s' because already executed", databaseName);
                return Boolean.FALSE;
            }
            iManager.getConfigurationMap().put(DEPLOYDB + databaseName, random);
            iManager.setDatabaseStatus(getNodeSource(), databaseName, ODistributedServerManager.DB_STATUS.SYNCHRONIZING);
            // PROPAGATE THE UPDATE TO ALL THE NODES
            //        iManager.sendRequest(databaseName, null, iManager.getActiveServers(),
            //            new OUpdateDatabaseStatusTask(databaseName, ODistributedServerManager.DB_STATUS.SYNCHRONIZING.name()),
            //            iManager.getNextMessageIdCounter(), ODistributedRequest.EXECUTION_MODE.RESPONSE, null, null);
            ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Deploying database %s...", databaseName);
            final AtomicReference<ODistributedMomentum> momentum = new AtomicReference<ODistributedMomentum>();
            File backupFile = ((ODistributedStorage) database.getStorage()).getLastValidBackup();
            if (backupFile == null || !backupFile.exists()) {
                // CREATE A BACKUP OF DATABASE FROM SCRATCH
                backupFile = new File(Orient.getTempPath() + "/backup_" + database.getName() + ".zip");
                final int compressionRate = OGlobalConfiguration.DISTRIBUTED_DEPLOYDB_TASK_COMPRESSION.getValueAsInteger();
                if (backupFile.exists())
                    backupFile.delete();
                else
                    backupFile.getParentFile().mkdirs();
                backupFile.createNewFile();
                final FileOutputStream fileOutputStream = new FileOutputStream(backupFile);
                final File completedFile = new File(backupFile.getAbsolutePath() + ".completed");
                if (completedFile.exists())
                    completedFile.delete();
                ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Creating backup of database '%s' (compressionRate=%d) in directory: %s...", databaseName, compressionRate, backupFile.getAbsolutePath());
                new Thread(new Runnable() {

                    @Override
                    public void run() {
                        Thread.currentThread().setName("OrientDB SyncDatabase node=" + iManager.getLocalNodeName() + " db=" + databaseName);
                        try {
                            database.activateOnCurrentThread();
                            ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Compressing database '%s' %d clusters %s...", databaseName, database.getClusterNames().size(), database.getClusterNames());
                            database.backup(fileOutputStream, null, new Callable<Object>() {

                                @Override
                                public Object call() throws Exception {
                                    momentum.set(dDatabase.getSyncConfiguration().getMomentum().copy());
                                    return null;
                                }
                            }, ODistributedServerLog.isDebugEnabled() ? new OCommandOutputListener() {

                                @Override
                                public void onMessage(String iText) {
                                    if (iText.startsWith("\n"))
                                        iText = iText.substring(1);
                                    OLogManager.instance().debug(this, iText);
                                }
                            } : null, OGlobalConfiguration.DISTRIBUTED_DEPLOYDB_TASK_COMPRESSION.getValueAsInteger(), CHUNK_MAX_SIZE);
                            ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Backup of database '%s' completed. lastOperationId=%s...", databaseName, requestId);
                        } catch (Throwable e) {
                            OLogManager.instance().error(this, "Cannot execute backup of database '%s' for deploy database", e, databaseName);
                        } finally {
                            try {
                                fileOutputStream.close();
                            } catch (IOException e) {
                            }
                            try {
                                completedFile.createNewFile();
                            } catch (IOException e) {
                                OLogManager.instance().error(this, "Cannot create file of backup completed: %s", e, completedFile);
                            }
                        }
                    }
                }).start();
                // RECORD LAST BACKUP TO BE REUSED IN CASE ANOTHER NODE ASK FOR THE SAME IN SHORT TIME WHILE THE DB IS NOT UPDATED
                ((ODistributedStorage) database.getStorage()).setLastValidBackup(backupFile);
            } else {
                momentum.set(dDatabase.getSyncConfiguration().getMomentum().copy());
                ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Reusing last backup of database '%s' in directory: %s...", databaseName, backupFile.getAbsolutePath());
            }
            for (int retry = 0; momentum.get() == null && retry < 10; ++retry) Thread.sleep(300);
            final ODistributedDatabaseChunk chunk = new ODistributedDatabaseChunk(backupFile, 0, CHUNK_MAX_SIZE, momentum.get(), false);
            ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), ODistributedServerLog.DIRECTION.OUT, "- transferring chunk #%d offset=%d size=%s lsn=%s...", 1, 0, OFileUtils.getSizeAsNumber(chunk.buffer.length), momentum.get());
            if (chunk.last)
                // NO MORE CHUNKS: SET THE NODE ONLINE (SYNCHRONIZING ENDED)
                iManager.setDatabaseStatus(iManager.getLocalNodeName(), databaseName, ODistributedServerManager.DB_STATUS.ONLINE);
            return chunk;
        } catch (OLockException e) {
            ODistributedServerLog.debug(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.NONE, "Skip deploying database %s because another node is doing it", databaseName);
        } finally {
            ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), ODistributedServerLog.DIRECTION.OUT, "Deploy database task completed");
        }
    } else
        ODistributedServerLog.debug(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.NONE, "Skip deploying database from the same node");
    return Boolean.FALSE;
}

Example 3 with OCommandOutputListener

use of com.orientechnologies.orient.core.command.OCommandOutputListener in project orientdb by orientechnologies.

the class ORemoteImportTest method testImport.

@Test
public void testImport() throws UnsupportedEncodingException {
    ODatabaseDocumentInternal db = new ODatabaseDocumentTx("remote:localhost/" + ORemoteImportTest.class.getSimpleName());
    db.open("admin", "admin");
    try {
        String content = "{\"records\": [{\"@type\": \"d\", \"@rid\": \"#9:0\",\"@version\": 1,\"@class\": \"V\"}]}";
        OStorageRemote storage = (OStorageRemote) db.getStorage();
        final StringBuffer buff = new StringBuffer();
        storage.importDatabase("-merge=true", new ByteArrayInputStream(content.getBytes("UTF8")), "data.json", new OCommandOutputListener() {

            @Override
            public void onMessage(String iText) {
                buff.append(iText);
            }
        });
        assertTrue(buff.toString().contains("Database import completed"));
    } finally {
        db.close();
    }
}

Example 4 with OCommandOutputListener

use of com.orientechnologies.orient.core.command.OCommandOutputListener in project orientdb by orientechnologies.

the class OConsoleDatabaseApp method repairDatabase.

@ConsoleCommand(description = "Repair database structure")
public void repairDatabase(@ConsoleParameter(name = "options", description = "Options: -v", optional = true) final String iOptions) throws IOException {
    checkForDatabase();
    message("\nRepairing database...");
    boolean verbose = iOptions != null && iOptions.contains("-v");
    new ODatabaseRepair().setDatabase(currentDatabase).setOutputListener(new OCommandOutputListener() {

        @Override
        public void onMessage(String iText) {
            message(iText);
        }
    }).setVerbose(verbose).run();
}

Example 5 with OCommandOutputListener

use of com.orientechnologies.orient.core.command.OCommandOutputListener in project orientdb by orientechnologies.

the class OAutomaticBackup method exportDatabase.

protected void exportDatabase(final String dbURL, final String iPath, final ODatabaseDocumentInternal db) throws IOException {
    OLogManager.instance().info(this, "AutomaticBackup: executing export of database '%s' to %s", dbURL, iPath);
    final ODatabaseExport exp = new ODatabaseExport(db, iPath, new OCommandOutputListener() {

        @Override
        public void onMessage(String iText) {
            OLogManager.instance().info(this, iText);
        }
    });
    if (exportOptions != null && !exportOptions.trim().isEmpty())
        exp.setOptions(exportOptions.trim());
    exp.exportDatabase().close();
}