Examples with OAbstractPaginatedStorage com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage used on opensource projects

Example 11 with OAbstractPaginatedStorage

use of com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage in project orientdb by orientechnologies.

the class OCommandExecutorSQLHASyncCluster method replaceCluster.

public static Object replaceCluster(final ODistributedAbstractPlugin dManager, final OServer serverInstance, final String databaseName, final String clusterName) {
    final ODistributedConfiguration cfg = dManager.getDatabaseConfiguration(databaseName);
    final String dbPath = serverInstance.getDatabaseDirectory() + databaseName;
    final String nodeName = dManager.getLocalNodeName();
    final List<String> nodesWhereClusterIsCfg = cfg.getServers(clusterName, null);
    nodesWhereClusterIsCfg.remove(nodeName);
    if (nodesWhereClusterIsCfg.isEmpty())
        throw new OCommandExecutionException("Cannot synchronize cluster '" + clusterName + "' because is not configured on any running nodes");
    final OSyncClusterTask task = new OSyncClusterTask(clusterName);
    final ODistributedResponse response = dManager.sendRequest(databaseName, null, nodesWhereClusterIsCfg, task, dManager.getNextMessageIdCounter(), ODistributedRequest.EXECUTION_MODE.RESPONSE, null, null);
    final Map<String, Object> results = (Map<String, Object>) response.getPayload();
    File tempFile = null;
    FileOutputStream out = null;
    try {
        tempFile = new File(Orient.getTempPath() + "/backup_" + databaseName + "_" + clusterName + "_toInstall.zip");
        if (tempFile.exists())
            tempFile.delete();
        else
            tempFile.getParentFile().mkdirs();
        tempFile.createNewFile();
        long fileSize = 0;
        out = new FileOutputStream(tempFile, false);
        for (Map.Entry<String, Object> r : results.entrySet()) {
            final Object value = r.getValue();
            if (value instanceof Boolean) {
                continue;
            } else if (value instanceof Throwable) {
                ODistributedServerLog.error(null, nodeName, r.getKey(), ODistributedServerLog.DIRECTION.IN, "error on installing cluster %s in %s", (Exception) value, databaseName, dbPath);
            } else if (value instanceof ODistributedDatabaseChunk) {
                ODistributedDatabaseChunk chunk = (ODistributedDatabaseChunk) value;
                // DELETE ANY PREVIOUS .COMPLETED FILE
                final File completedFile = new File(tempFile.getAbsolutePath() + ".completed");
                if (completedFile.exists())
                    completedFile.delete();
                fileSize = writeDatabaseChunk(nodeName, 1, chunk, out);
                for (int chunkNum = 2; !chunk.last; chunkNum++) {
                    final Object result = dManager.sendRequest(databaseName, null, OMultiValue.getSingletonList(r.getKey()), new OCopyDatabaseChunkTask(chunk.filePath, chunkNum, chunk.offset + chunk.buffer.length, false), dManager.getNextMessageIdCounter(), ODistributedRequest.EXECUTION_MODE.RESPONSE, null, null);
                    if (result instanceof Boolean)
                        continue;
                    else if (result instanceof Exception) {
                        ODistributedServerLog.error(null, nodeName, r.getKey(), ODistributedServerLog.DIRECTION.IN, "error on installing database %s in %s (chunk #%d)", (Exception) result, databaseName, dbPath, chunkNum);
                    } else if (result instanceof ODistributedDatabaseChunk) {
                        chunk = (ODistributedDatabaseChunk) result;
                        fileSize += writeDatabaseChunk(nodeName, chunkNum, chunk, out);
                    }
                }
                out.flush();
                // CREATE THE .COMPLETED FILE TO SIGNAL EOF
                new File(tempFile.getAbsolutePath() + ".completed").createNewFile();
            }
        }
        final String tempDirectoryPath = Orient.getTempPath() + "/backup_" + databaseName + "_" + clusterName + "_toInstall";
        final File tempDirectory = new File(tempDirectoryPath);
        tempDirectory.mkdirs();
        OZIPCompressionUtil.uncompressDirectory(new FileInputStream(tempFile), tempDirectory.getAbsolutePath(), null);
        ODatabaseDocumentInternal db = ODatabaseRecordThreadLocal.INSTANCE.getIfDefined();
        final boolean openDatabaseHere = db == null;
        if (db == null)
            db = serverInstance.openDatabase("plocal:" + dbPath, "", "", null, true);
        try {
            final OAbstractPaginatedStorage stg = (OAbstractPaginatedStorage) db.getStorage().getUnderlying();
            // TODO: FREEZE COULD IT NOT NEEDED
            stg.freeze(false);
            try {
                final OPaginatedCluster cluster = (OPaginatedCluster) stg.getClusterByName(clusterName);
                final File tempClusterFile = new File(tempDirectoryPath + "/" + clusterName + OPaginatedCluster.DEF_EXTENSION);
                cluster.replaceFile(tempClusterFile);
            } finally {
                stg.release();
            }
            db.getLocalCache().invalidate();
        } finally {
            if (openDatabaseHere)
                db.close();
        }
        return String.format("Cluster correctly replaced, transferred %d bytes", fileSize);
    } catch (Exception e) {
        ODistributedServerLog.error(null, nodeName, null, ODistributedServerLog.DIRECTION.NONE, "error on transferring database '%s' to '%s'", e, databaseName, tempFile);
        throw OException.wrapException(new ODistributedException("Error on transferring database"), e);
    } finally {
        try {
            if (out != null) {
                out.flush();
                out.close();
            }
        } catch (IOException e) {
        }
    }
}

Example 12 with OAbstractPaginatedStorage

use of com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage in project orientdb by orientechnologies.

the class OSyncDatabaseDeltaTask method deltaBackup.

protected Object deltaBackup(final ODistributedRequestId requestId, final ODistributedServerManager iManager, final ODatabaseDocumentInternal database, final String databaseName) throws IOException, InterruptedException {
    final Long lastDeployment = (Long) iManager.getConfigurationMap().get(DEPLOYDB + databaseName);
    if (lastDeployment != null && lastDeployment.longValue() == random) {
        // SKIP IT
        ODistributedServerLog.debug(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.NONE, "Skip deploying delta database '%s' because already executed", databaseName);
        return Boolean.FALSE;
    }
    iManager.getConfigurationMap().put(DEPLOYDB + databaseName, random);
    final ODistributedDatabase dDatabase = checkIfCurrentDatabaseIsNotOlder(iManager, databaseName, startLSN);
    iManager.setDatabaseStatus(getNodeSource(), databaseName, ODistributedServerManager.DB_STATUS.SYNCHRONIZING);
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Deploying database '%s' with delta of changes...", databaseName);
    // CREATE A BACKUP OF DATABASE
    final File backupFile = new File(Orient.getTempPath() + "/backup_" + getNodeSource() + "_" + database.getName() + ".zip");
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Creating delta backup of database '%s' (startLSN=%s) in directory: %s...", databaseName, startLSN, backupFile.getAbsolutePath());
    if (backupFile.exists())
        backupFile.delete();
    else
        backupFile.getParentFile().mkdirs();
    backupFile.createNewFile();
    final FileOutputStream fileOutputStream = new FileOutputStream(backupFile);
    // final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream);
    final File completedFile = new File(backupFile.getAbsolutePath() + ".completed");
    if (completedFile.exists())
        completedFile.delete();
    final OStorage storage = database.getStorage().getUnderlying();
    if (!(storage instanceof OAbstractPaginatedStorage))
        throw new UnsupportedOperationException("Storage '" + storage.getName() + "' does not support distributed delta backup");
    final AtomicReference<OLogSequenceNumber> endLSN = new AtomicReference<OLogSequenceNumber>();
    final AtomicReference<ODistributedDatabaseDeltaSyncException> exception = new AtomicReference<ODistributedDatabaseDeltaSyncException>();
    try {
        final AtomicLong counter = new AtomicLong(0);
        endLSN.set(((OAbstractPaginatedStorage) storage).recordsChangedAfterLSN(startLSN, fileOutputStream, excludedClusterNames, new OCommandOutputListener() {

            @Override
            public void onMessage(final String iText) {
                if (iText.startsWith("read")) {
                    if (counter.incrementAndGet() % 100000 == 0) {
                        ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "- %s", iText);
                    }
                } else if (counter.incrementAndGet() % 10000 == 0) {
                    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "- %s", iText);
                }
            }
        }));
        if (endLSN.get() == null) {
            // DELTA NOT AVAILABLE, TRY WITH FULL BACKUP
            exception.set(new ODistributedDatabaseDeltaSyncException(startLSN));
        } else
            ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Delta backup of database '%s' completed. range=%s-%s", databaseName, startLSN, endLSN.get());
    } catch (Exception e) {
        // UNKNOWN ERROR, DELTA NOT AVAILABLE, TRY WITH FULL BACKUP
        exception.set(new ODistributedDatabaseDeltaSyncException(startLSN, e.getMessage()));
    } finally {
        try {
            fileOutputStream.close();
        } catch (IOException e) {
        }
        try {
            completedFile.createNewFile();
        } catch (IOException e) {
            OLogManager.instance().error(this, "Cannot create file of delta backup completed: %s", e, completedFile);
        }
    }
    if (exception.get() instanceof ODistributedDatabaseDeltaSyncException) {
        throw exception.get();
    }
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "Deploy delta database task completed");
    // GET THE MOMENTUM, BUT OVERWRITE THE LAST LSN RECEIVED FROM THE DELTA
    final ODistributedMomentum momentum = dDatabase.getSyncConfiguration().getMomentum().copy();
    momentum.setLSN(iManager.getLocalNodeName(), endLSN.get());
    final ODistributedDatabaseChunk chunk = new ODistributedDatabaseChunk(backupFile, 0, CHUNK_MAX_SIZE, momentum, false);
    ODistributedServerLog.info(this, iManager.getLocalNodeName(), getNodeSource(), DIRECTION.OUT, "- transferring chunk #%d offset=%d size=%s...", 1, 0, OFileUtils.getSizeAsNumber(chunk.buffer.length));
    if (chunk.last)
        // NO MORE CHUNKS: SET THE NODE ONLINE (SYNCHRONIZING ENDED)
        iManager.setDatabaseStatus(iManager.getLocalNodeName(), databaseName, ODistributedServerManager.DB_STATUS.ONLINE);
    return chunk;
}

Example 13 with OAbstractPaginatedStorage

use of com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage in project orientdb by orientechnologies.

the class OLuceneStorage method reOpen.

private void reOpen() throws IOException {
    if (mgrWriter != null) {
        OLogManager.instance().info(this, "index storage is open don't reopen");
        return;
    }
    ODatabaseDocumentInternal database = ODatabaseRecordThreadLocal.INSTANCE.get();
    final OAbstractPaginatedStorage storageLocalAbstract = (OAbstractPaginatedStorage) database.getStorage().getUnderlying();
    Directory dir = null;
    if (storageLocalAbstract instanceof OLocalPaginatedStorage) {
        String pathname = getIndexPath((OLocalPaginatedStorage) storageLocalAbstract);
        OLogManager.instance().info(this, "Opening NIOFS Lucene db=%s, path=%s", database.getName(), pathname);
        dir = NIOFSDirectory.open(new File(pathname).toPath());
    } else {
        OLogManager.instance().info(this, "Opening RAM Lucene index db=%s", database.getName());
        dir = new RAMDirectory();
    }
    final IndexWriter indexWriter = createIndexWriter(dir);
    mgrWriter = new TrackingIndexWriter(indexWriter);
    searcherManager = new SearcherManager(indexWriter, true, null);
    if (nrt != null) {
        nrt.close();
    }
    nrt = new ControlledRealTimeReopenThread(mgrWriter, searcherManager, 60.00, 0.1);
    nrt.setDaemon(true);
    nrt.start();
    flush();
    OLogManager.instance().info(this, "REOPEN DONE");
}

Example 14 with OAbstractPaginatedStorage

use of com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage in project orientdb by orientechnologies.

the class OLuceneStorage method delete.

public void delete(final ODatabaseInternal database) {
    OLogManager.instance().info(this, "DELETING STORAGE:: ");
    close();
    final OAbstractPaginatedStorage storageLocalAbstract = (OAbstractPaginatedStorage) database.getStorage().getUnderlying();
    if (storageLocalAbstract instanceof OLocalPaginatedStorage) {
        String pathname = getIndexPath((OLocalPaginatedStorage) storageLocalAbstract);
        OFileUtils.deleteRecursively(new File(pathname));
    }
}

Example 15 with OAbstractPaginatedStorage

use of com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage in project orientdb by orientechnologies.

the class OAutoShardingIndexEngine method load.

@Override
public void load(final String indexName, final OBinarySerializer valueSerializer, final boolean isAutomatic, final OBinarySerializer keySerializer, final OType[] keyTypes, final boolean nullPointerSupport, final int keySize, final Map<String, String> engineProperties) {
    this.strategy = new OAutoShardingMurmurStrategy(keySerializer);
    final OStorage storage = getDatabase().getStorage().getUnderlying();
    if (storage instanceof OAbstractPaginatedStorage) {
        final String partitionsAsString = engineProperties.get("partitions");
        if (partitionsAsString == null || partitionsAsString.isEmpty())
            throw new OIndexException("Cannot load autosharding index '" + indexName + "' because there is no metadata about the number of partitions");
        partitionSize = Integer.parseInt(partitionsAsString);
        init();
        int i = 0;
        for (OHashTable<Object, Object> p : partitions) p.load(indexName + "_" + (i++), keyTypes, nullPointerSupport);
    }
    hashFunction.setValueSerializer(keySerializer);
}