Examples with MongoConnection com.eightkdata.mongowp.client.core.MongoConnection used on opensource projects

Example 1 with MongoConnection

use of com.eightkdata.mongowp.client.core.MongoConnection in project torodb by torodb.

the class AbstractMongoOplogReader method getFirstOrLastOp.

private OplogOperation getFirstOrLastOp(boolean first) throws OplogStartMissingException, OplogOperationUnsupported, MongoException {
    Preconditions.checkState(!isClosed(), "You have to connect this client before");
    BsonDocument query = DefaultBsonValues.EMPTY_DOC;
    BsonDocument orderBy = first ? NATURAL_ORDER_SORT : INVERSE_ORDER_SORT;
    EnumSet<QueryOption> flags = EnumSet.of(QueryOption.SLAVE_OK);
    BsonDocument doc;
    MongoConnection connection = consumeConnection();
    try {
        MongoCursor<BsonDocument> cursor = connection.query(DATABASE, COLLECTION, query, 0, 1, new QueryOptions(flags), orderBy, null);
        try {
            Batch<BsonDocument> batch = cursor.fetchBatch();
            try {
                if (!batch.hasNext()) {
                    throw new OplogStartMissingException(getSyncSource());
                }
                doc = batch.next();
            } finally {
                batch.close();
            }
        } finally {
            cursor.close();
        }
        try {
            return OplogOperationParser.fromBson(doc);
        } catch (BadValueException | TypesMismatchException | NoSuchKeyException ex) {
            throw new OplogOperationUnsupported(doc, ex);
        }
    } finally {
        releaseConnection(connection);
    }
}

Example 2 with MongoConnection

use of com.eightkdata.mongowp.client.core.MongoConnection in project torodb by torodb.

the class RecoveryService method initialSync.

private boolean initialSync() throws TryAgainException, FatalErrorException {
    /*
     * 1. store that data is inconsistent 2. decide a sync source 3. lastRemoteOptime1 = get the
     * last optime of the sync source 4. clone all databases except local 5. lastRemoteOptime2 = get
     * the last optime of the sync source 6. apply remote oplog from lastRemoteOptime1 to
     * lastRemoteOptime2 7. lastRemoteOptime3 = get the last optime of the sync source 8. apply
     * remote oplog from lastRemoteOptime2 to lastRemoteOptime3 9. rebuild indexes 10. store
     * lastRemoteOptime3 as the last applied operation optime 11. store that data is consistent 12.
     * change replication state to SECONDARY
     */
    //TODO: Support fastsync (used to restore a node by copying the data from other up-to-date node)
    LOGGER.info("Starting initial sync");
    callback.setConsistentState(false);
    HostAndPort syncSource;
    try {
        syncSource = syncSourceProvider.newSyncSource();
        LOGGER.info("Using node " + syncSource + " to replicate from");
    } catch (NoSyncSourceFoundException ex) {
        throw new TryAgainException("No sync source");
    }
    MongoClient remoteClient;
    try {
        remoteClient = remoteClientFactory.createClient(syncSource);
    } catch (UnreachableMongoServerException ex) {
        throw new TryAgainException(ex);
    }
    try {
        LOGGER.debug("Remote client obtained");
        MongoConnection remoteConnection = remoteClient.openConnection();
        try (OplogReader reader = oplogReaderProvider.newReader(remoteConnection)) {
            OplogOperation lastClonedOp = reader.getLastOp();
            OpTime lastRemoteOptime1 = lastClonedOp.getOpTime();
            try (WriteOplogTransaction oplogTransaction = oplogManager.createWriteTransaction()) {
                LOGGER.info("Remote database cloning started");
                oplogTransaction.truncate();
                LOGGER.info("Local databases dropping started");
                Status<?> status = dropDatabases();
                if (!status.isOk()) {
                    throw new TryAgainException("Error while trying to drop collections: " + status);
                }
                LOGGER.info("Local databases dropping finished");
                if (!isRunning()) {
                    LOGGER.warn("Recovery stopped before it can finish");
                    return false;
                }
                LOGGER.info("Remote database cloning started");
                cloneDatabases(remoteClient);
                LOGGER.info("Remote database cloning finished");
                oplogTransaction.forceNewValue(lastClonedOp.getHash(), lastClonedOp.getOpTime());
            }
            if (!isRunning()) {
                LOGGER.warn("Recovery stopped before it can finish");
                return false;
            }
            TorodServer torodServer = server.getTorodServer();
            try (TorodConnection connection = torodServer.openConnection();
                SharedWriteTorodTransaction trans = connection.openWriteTransaction(false)) {
                OpTime lastRemoteOptime2 = reader.getLastOp().getOpTime();
                LOGGER.info("First oplog application started");
                applyOplog(reader, lastRemoteOptime1, lastRemoteOptime2);
                trans.commit();
                LOGGER.info("First oplog application finished");
                if (!isRunning()) {
                    LOGGER.warn("Recovery stopped before it can finish");
                    return false;
                }
                OplogOperation lastOperation = reader.getLastOp();
                OpTime lastRemoteOptime3 = lastOperation.getOpTime();
                LOGGER.info("Second oplog application started");
                applyOplog(reader, lastRemoteOptime2, lastRemoteOptime3);
                trans.commit();
                LOGGER.info("Second oplog application finished");
                if (!isRunning()) {
                    LOGGER.warn("Recovery stopped before it can finish");
                    return false;
                }
                LOGGER.info("Index rebuild started");
                rebuildIndexes();
                trans.commit();
                LOGGER.info("Index rebuild finished");
                if (!isRunning()) {
                    LOGGER.warn("Recovery stopped before it can finish");
                    return false;
                }
                trans.commit();
            }
        } catch (OplogStartMissingException ex) {
            throw new TryAgainException(ex);
        } catch (OplogOperationUnsupported ex) {
            throw new TryAgainException(ex);
        } catch (MongoException | RollbackException ex) {
            throw new TryAgainException(ex);
        } catch (OplogManagerPersistException ex) {
            throw new FatalErrorException();
        } catch (UserException ex) {
            throw new FatalErrorException(ex);
        }
        callback.setConsistentState(true);
        LOGGER.info("Initial sync finished");
    } finally {
        remoteClient.close();
    }
    return true;
}

Example 3 with MongoConnection

use of com.eightkdata.mongowp.client.core.MongoConnection in project torodb by torodb.

the class RecoveryService method cloneDatabases.

private void cloneDatabases(@Nonnull MongoClient remoteClient) throws CloningException, MongoException, UserException {
    enableDataImportMode();
    try {
        Stream<String> dbNames;
        try (MongoConnection remoteConnection = remoteClient.openConnection()) {
            RemoteCommandResponse<ListDatabasesReply> remoteResponse = remoteConnection.execute(ListDatabasesCommand.INSTANCE, "admin", true, Empty.getInstance());
            if (!remoteResponse.isOk()) {
                throw remoteResponse.asMongoException();
            }
            dbNames = remoteResponse.getCommandReply().get().getDatabases().stream().map(db -> db.getName());
        }
        dbNames.filter(this::isReplicable).forEach(databaseName -> {
            MyWritePermissionSupplier writePermissionSupplier = new MyWritePermissionSupplier(databaseName);
            CloneOptions options = new CloneOptions(true, true, true, false, databaseName, Collections.<String>emptySet(), writePermissionSupplier, (colName) -> replFilters.getCollectionPredicate().test(databaseName, colName), (collection, indexName, unique, keys) -> replFilters.getIndexPredicate().test(databaseName, collection, indexName, unique, keys));
            try {
                cloner.cloneDatabase(databaseName, remoteClient, server, options);
            } catch (MongoException ex) {
                throw new CloningException(ex);
            }
        });
    } finally {
        disableDataImportMode();
    }
}

Example 4 with MongoConnection

use of com.eightkdata.mongowp.client.core.MongoConnection in project torodb by torodb.

the class AbstractMongoOplogReader method query.

public MongoCursor<OplogOperation> query(BsonDocument query, EnumSet<QueryOption> flags, BsonDocument sortBy) throws MongoException {
    Preconditions.checkState(!isClosed(), "You have to connect this client before");
    MongoConnection connection = consumeConnection();
    MongoCursor<BsonDocument> cursor = connection.query(DATABASE, COLLECTION, query, 0, 0, new QueryOptions(flags), sortBy, null);
    return new MyCursor<>(connection, TransformationMongoCursor.create(cursor, OplogOperationParser.asFunction()));
}

Example 5 with MongoConnection

use of com.eightkdata.mongowp.client.core.MongoConnection in project torodb by torodb.

the class AkkaDbCloner method cloneDatabase.

@Override
public void cloneDatabase(String dstDb, MongoClient remoteClient, MongodServer localServer, CloneOptions opts) throws CloningException, NotMasterException, MongoException {
    Preconditions.checkState(isRunning(), "This db cloner is not running");
    if (!remoteClient.isRemote() && opts.getDbToClone().equals(dstDb)) {
        LOGGER.warn("Trying to clone a database to itself! Ignoring it");
        return;
    }
    String fromDb = opts.getDbToClone();
    CursorResult<Entry> listCollections;
    try (MongoConnection remoteConnection = remoteClient.openConnection()) {
        listCollections = ListCollectionsRequester.getListCollections(remoteConnection, fromDb, null);
    } catch (MongoException ex) {
        throw new CloningException("It was impossible to get information from the remote server", ex);
    }
    if (!opts.getWritePermissionSupplier().get()) {
        throw new NotMasterException("Destiny database cannot be written");
    }
    List<Entry> collsToClone = getCollsToClone(listCollections, fromDb, opts);
    if (!opts.getWritePermissionSupplier().get()) {
        throw new NotMasterException("Destiny database cannot be written " + "after get collections info");
    }
    try {
        for (Entry entry : collsToClone) {
            prepareCollection(localServer, dstDb, entry);
        }
    } catch (RollbackException ex) {
        throw new AssertionError("Unexpected rollback exception", ex);
    }
    Materializer materializer = ActorMaterializer.create(getActorSystem());
    try (MongoConnection remoteConnection = remoteClient.openConnection()) {
        if (opts.isCloneData()) {
            for (Entry entry : collsToClone) {
                LOGGER.info("Cloning collection data {}.{} into {}.{}", fromDb, entry.getCollectionName(), dstDb, entry.getCollectionName());
                try {
                    cloneCollection(localServer, remoteConnection, dstDb, opts, materializer, entry);
                } catch (CompletionException completionException) {
                    Throwable cause = completionException.getCause();
                    if (cause instanceof RollbackException) {
                        throw (RollbackException) cause;
                    }
                    throw completionException;
                }
            }
        }
        if (opts.isCloneIndexes()) {
            for (Entry entry : collsToClone) {
                LOGGER.info("Cloning collection indexes {}.{} into {}.{}", fromDb, entry.getCollectionName(), dstDb, entry.getCollectionName());
                try {
                    cloneIndex(localServer, dstDb, dstDb, remoteConnection, opts, entry.getCollectionName(), entry.getCollectionName());
                } catch (CompletionException completionException) {
                    Throwable cause = completionException.getCause();
                    if (cause instanceof RollbackException) {
                        throw (RollbackException) cause;
                    }
                    throw completionException;
                }
            }
        }
    }
}