Search in sources :

Example 1 with MongoClient

use of com.eightkdata.mongowp.client.core.MongoClient in project torodb by torodb.

the class RecoveryService method initialSync.

private boolean initialSync() throws TryAgainException, FatalErrorException {
    /*
     * 1. store that data is inconsistent 2. decide a sync source 3. lastRemoteOptime1 = get the
     * last optime of the sync source 4. clone all databases except local 5. lastRemoteOptime2 = get
     * the last optime of the sync source 6. apply remote oplog from lastRemoteOptime1 to
     * lastRemoteOptime2 7. lastRemoteOptime3 = get the last optime of the sync source 8. apply
     * remote oplog from lastRemoteOptime2 to lastRemoteOptime3 9. rebuild indexes 10. store
     * lastRemoteOptime3 as the last applied operation optime 11. store that data is consistent 12.
     * change replication state to SECONDARY
     */
    //TODO: Support fastsync (used to restore a node by copying the data from other up-to-date node)
    LOGGER.info("Starting initial sync");
    callback.setConsistentState(false);
    HostAndPort syncSource;
    try {
        syncSource = syncSourceProvider.newSyncSource();
        LOGGER.info("Using node " + syncSource + " to replicate from");
    } catch (NoSyncSourceFoundException ex) {
        throw new TryAgainException("No sync source");
    }
    MongoClient remoteClient;
    try {
        remoteClient = remoteClientFactory.createClient(syncSource);
    } catch (UnreachableMongoServerException ex) {
        throw new TryAgainException(ex);
    }
    try {
        LOGGER.debug("Remote client obtained");
        MongoConnection remoteConnection = remoteClient.openConnection();
        try (OplogReader reader = oplogReaderProvider.newReader(remoteConnection)) {
            OplogOperation lastClonedOp = reader.getLastOp();
            OpTime lastRemoteOptime1 = lastClonedOp.getOpTime();
            try (WriteOplogTransaction oplogTransaction = oplogManager.createWriteTransaction()) {
                LOGGER.info("Remote database cloning started");
                oplogTransaction.truncate();
                LOGGER.info("Local databases dropping started");
                Status<?> status = dropDatabases();
                if (!status.isOk()) {
                    throw new TryAgainException("Error while trying to drop collections: " + status);
                }
                LOGGER.info("Local databases dropping finished");
                if (!isRunning()) {
                    LOGGER.warn("Recovery stopped before it can finish");
                    return false;
                }
                LOGGER.info("Remote database cloning started");
                cloneDatabases(remoteClient);
                LOGGER.info("Remote database cloning finished");
                oplogTransaction.forceNewValue(lastClonedOp.getHash(), lastClonedOp.getOpTime());
            }
            if (!isRunning()) {
                LOGGER.warn("Recovery stopped before it can finish");
                return false;
            }
            TorodServer torodServer = server.getTorodServer();
            try (TorodConnection connection = torodServer.openConnection();
                SharedWriteTorodTransaction trans = connection.openWriteTransaction(false)) {
                OpTime lastRemoteOptime2 = reader.getLastOp().getOpTime();
                LOGGER.info("First oplog application started");
                applyOplog(reader, lastRemoteOptime1, lastRemoteOptime2);
                trans.commit();
                LOGGER.info("First oplog application finished");
                if (!isRunning()) {
                    LOGGER.warn("Recovery stopped before it can finish");
                    return false;
                }
                OplogOperation lastOperation = reader.getLastOp();
                OpTime lastRemoteOptime3 = lastOperation.getOpTime();
                LOGGER.info("Second oplog application started");
                applyOplog(reader, lastRemoteOptime2, lastRemoteOptime3);
                trans.commit();
                LOGGER.info("Second oplog application finished");
                if (!isRunning()) {
                    LOGGER.warn("Recovery stopped before it can finish");
                    return false;
                }
                LOGGER.info("Index rebuild started");
                rebuildIndexes();
                trans.commit();
                LOGGER.info("Index rebuild finished");
                if (!isRunning()) {
                    LOGGER.warn("Recovery stopped before it can finish");
                    return false;
                }
                trans.commit();
            }
        } catch (OplogStartMissingException ex) {
            throw new TryAgainException(ex);
        } catch (OplogOperationUnsupported ex) {
            throw new TryAgainException(ex);
        } catch (MongoException | RollbackException ex) {
            throw new TryAgainException(ex);
        } catch (OplogManagerPersistException ex) {
            throw new FatalErrorException();
        } catch (UserException ex) {
            throw new FatalErrorException(ex);
        }
        callback.setConsistentState(true);
        LOGGER.info("Initial sync finished");
    } finally {
        remoteClient.close();
    }
    return true;
}
Also used : OplogManagerPersistException(com.torodb.mongodb.repl.OplogManager.OplogManagerPersistException) SharedWriteTorodTransaction(com.torodb.torod.SharedWriteTorodTransaction) MongoException(com.eightkdata.mongowp.exceptions.MongoException) OplogOperationUnsupported(com.eightkdata.mongowp.exceptions.OplogOperationUnsupported) UnreachableMongoServerException(com.eightkdata.mongowp.client.core.UnreachableMongoServerException) OplogStartMissingException(com.eightkdata.mongowp.exceptions.OplogStartMissingException) RollbackException(com.torodb.core.transaction.RollbackException) OpTime(com.eightkdata.mongowp.OpTime) WriteOplogTransaction(com.torodb.mongodb.repl.OplogManager.WriteOplogTransaction) HostAndPort(com.google.common.net.HostAndPort) MongoClient(com.eightkdata.mongowp.client.core.MongoClient) TorodServer(com.torodb.torod.TorodServer) UserException(com.torodb.core.exceptions.user.UserException) NoSyncSourceFoundException(com.torodb.mongodb.repl.exceptions.NoSyncSourceFoundException) OplogOperation(com.eightkdata.mongowp.server.api.oplog.OplogOperation) MongoConnection(com.eightkdata.mongowp.client.core.MongoConnection) TorodConnection(com.torodb.torod.TorodConnection)

Example 2 with MongoClient

use of com.eightkdata.mongowp.client.core.MongoClient in project torodb by torodb.

the class RecoveryService method cloneDatabases.

private void cloneDatabases(@Nonnull MongoClient remoteClient) throws CloningException, MongoException, UserException {
    enableDataImportMode();
    try {
        Stream<String> dbNames;
        try (MongoConnection remoteConnection = remoteClient.openConnection()) {
            RemoteCommandResponse<ListDatabasesReply> remoteResponse = remoteConnection.execute(ListDatabasesCommand.INSTANCE, "admin", true, Empty.getInstance());
            if (!remoteResponse.isOk()) {
                throw remoteResponse.asMongoException();
            }
            dbNames = remoteResponse.getCommandReply().get().getDatabases().stream().map(db -> db.getName());
        }
        dbNames.filter(this::isReplicable).forEach(databaseName -> {
            MyWritePermissionSupplier writePermissionSupplier = new MyWritePermissionSupplier(databaseName);
            CloneOptions options = new CloneOptions(true, true, true, false, databaseName, Collections.<String>emptySet(), writePermissionSupplier, (colName) -> replFilters.getCollectionPredicate().test(databaseName, colName), (collection, indexName, unique, keys) -> replFilters.getIndexPredicate().test(databaseName, collection, indexName, unique, keys));
            try {
                cloner.cloneDatabase(databaseName, remoteClient, server, options);
            } catch (MongoException ex) {
                throw new CloningException(ex);
            }
        });
    } finally {
        disableDataImportMode();
    }
}
Also used : RunnableTorodbService(com.torodb.core.services.RunnableTorodbService) ReadOplogTransaction(com.torodb.mongodb.repl.OplogManager.ReadOplogTransaction) MongoConnection(com.eightkdata.mongowp.client.core.MongoConnection) RollbackException(com.torodb.core.transaction.RollbackException) Assisted(com.google.inject.assistedinject.Assisted) OplogOperation(com.eightkdata.mongowp.server.api.oplog.OplogOperation) MongoDbRepl(com.torodb.mongodb.repl.guice.MongoDbRepl) MongoClient(com.eightkdata.mongowp.client.core.MongoClient) Empty(com.eightkdata.mongowp.server.api.tools.Empty) RollbackReplicationException(com.torodb.mongodb.repl.oplogreplier.RollbackReplicationException) MongoException(com.eightkdata.mongowp.exceptions.MongoException) OplogOperationUnsupported(com.eightkdata.mongowp.exceptions.OplogOperationUnsupported) ThreadFactory(java.util.concurrent.ThreadFactory) OplogFetcher(com.torodb.mongodb.repl.oplogreplier.fetcher.OplogFetcher) CancellationException(java.util.concurrent.CancellationException) CloningException(com.torodb.mongodb.utils.DbCloner.CloningException) ListDatabasesCommand(com.torodb.mongodb.commands.signatures.diagnostic.ListDatabasesCommand) OplogApplier(com.torodb.mongodb.repl.oplogreplier.OplogApplier) List(java.util.List) Logger(org.apache.logging.log4j.Logger) Stream(java.util.stream.Stream) StopReplicationException(com.torodb.mongodb.repl.oplogreplier.StopReplicationException) OpTime(com.eightkdata.mongowp.OpTime) OplogStartMissingException(com.eightkdata.mongowp.exceptions.OplogStartMissingException) Supplier(com.google.common.base.Supplier) MongodServer(com.torodb.mongodb.core.MongodServer) UnreachableMongoServerException(com.eightkdata.mongowp.client.core.UnreachableMongoServerException) Inject(javax.inject.Inject) TorodConnection(com.torodb.torod.TorodConnection) NoSyncSourceFoundException(com.torodb.mongodb.repl.exceptions.NoSyncSourceFoundException) ApplierContext(com.torodb.mongodb.repl.oplogreplier.ApplierContext) MongoClientFactory(com.eightkdata.mongowp.client.core.MongoClientFactory) LimitedOplogFetcher(com.torodb.mongodb.repl.oplogreplier.fetcher.LimitedOplogFetcher) RemoteCommandResponse(com.eightkdata.mongowp.client.core.MongoConnection.RemoteCommandResponse) OplogManagerPersistException(com.torodb.mongodb.repl.OplogManager.OplogManagerPersistException) DbCloner(com.torodb.mongodb.utils.DbCloner) Supervisor(com.torodb.core.supervision.Supervisor) Nonnull(javax.annotation.Nonnull) CloneOptions(com.torodb.mongodb.utils.DbCloner.CloneOptions) SharedWriteTorodTransaction(com.torodb.torod.SharedWriteTorodTransaction) TorodbRunnableService(com.torodb.core.annotations.TorodbRunnableService) SupervisorDecision(com.torodb.core.supervision.SupervisorDecision) MongoCursor(com.eightkdata.mongowp.server.api.pojos.MongoCursor) UnexpectedOplogApplierException(com.torodb.mongodb.repl.oplogreplier.OplogApplier.UnexpectedOplogApplierException) UserException(com.torodb.core.exceptions.user.UserException) WriteOplogTransaction(com.torodb.mongodb.repl.OplogManager.WriteOplogTransaction) HostAndPort(com.google.common.net.HostAndPort) Status(com.eightkdata.mongowp.Status) TorodServer(com.torodb.torod.TorodServer) ListDatabasesReply(com.torodb.mongodb.commands.signatures.diagnostic.ListDatabasesCommand.ListDatabasesReply) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) CloningException(com.torodb.mongodb.utils.DbCloner.CloningException) MongoException(com.eightkdata.mongowp.exceptions.MongoException) ListDatabasesReply(com.torodb.mongodb.commands.signatures.diagnostic.ListDatabasesCommand.ListDatabasesReply) MongoConnection(com.eightkdata.mongowp.client.core.MongoConnection) CloneOptions(com.torodb.mongodb.utils.DbCloner.CloneOptions)

Aggregations

OpTime (com.eightkdata.mongowp.OpTime)2 MongoClient (com.eightkdata.mongowp.client.core.MongoClient)2 MongoConnection (com.eightkdata.mongowp.client.core.MongoConnection)2 UnreachableMongoServerException (com.eightkdata.mongowp.client.core.UnreachableMongoServerException)2 MongoException (com.eightkdata.mongowp.exceptions.MongoException)2 OplogOperationUnsupported (com.eightkdata.mongowp.exceptions.OplogOperationUnsupported)2 OplogStartMissingException (com.eightkdata.mongowp.exceptions.OplogStartMissingException)2 OplogOperation (com.eightkdata.mongowp.server.api.oplog.OplogOperation)2 HostAndPort (com.google.common.net.HostAndPort)2 UserException (com.torodb.core.exceptions.user.UserException)2 RollbackException (com.torodb.core.transaction.RollbackException)2 OplogManagerPersistException (com.torodb.mongodb.repl.OplogManager.OplogManagerPersistException)2 WriteOplogTransaction (com.torodb.mongodb.repl.OplogManager.WriteOplogTransaction)2 NoSyncSourceFoundException (com.torodb.mongodb.repl.exceptions.NoSyncSourceFoundException)2 SharedWriteTorodTransaction (com.torodb.torod.SharedWriteTorodTransaction)2 TorodConnection (com.torodb.torod.TorodConnection)2 TorodServer (com.torodb.torod.TorodServer)2 Status (com.eightkdata.mongowp.Status)1 MongoClientFactory (com.eightkdata.mongowp.client.core.MongoClientFactory)1 RemoteCommandResponse (com.eightkdata.mongowp.client.core.MongoConnection.RemoteCommandResponse)1