Examples with BackupRepository org.apache.solr.core.backup.repository.BackupRepository used on opensource projects

Example 1 with BackupRepository

use of org.apache.solr.core.backup.repository.BackupRepository in project lucene-solr by apache.

the class ReplicationHandler method restore.

private void restore(SolrParams params, SolrQueryResponse rsp, SolrQueryRequest req) throws IOException {
    if (restoreFuture != null && !restoreFuture.isDone()) {
        throw new SolrException(ErrorCode.BAD_REQUEST, "Restore in progress. Cannot run multiple restore operations" + "for the same core");
    }
    String name = params.get(NAME);
    String location = params.get(CoreAdminParams.BACKUP_LOCATION);
    String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY);
    CoreContainer cc = core.getCoreContainer();
    BackupRepository repo = null;
    if (repoName != null) {
        repo = cc.newBackupRepository(Optional.of(repoName));
        location = repo.getBackupLocation(location);
        if (location == null) {
            throw new IllegalArgumentException("location is required");
        }
    } else {
        repo = new LocalFileSystemRepository();
    }
    //If location is not provided then assume that the restore index is present inside the data directory.
    if (location == null) {
        location = core.getDataDir();
    }
    URI locationUri = repo.createURI(location);
    //snapshot folder since we allow snapshots to be taken without providing a name. Pick the latest timestamp.
    if (name == null) {
        String[] filePaths = repo.listAll(locationUri);
        List<OldBackupDirectory> dirs = new ArrayList<>();
        for (String f : filePaths) {
            OldBackupDirectory obd = new OldBackupDirectory(locationUri, f);
            if (obd.getTimestamp().isPresent()) {
                dirs.add(obd);
            }
        }
        Collections.sort(dirs);
        if (dirs.size() == 0) {
            throw new SolrException(ErrorCode.BAD_REQUEST, "No backup name specified and none found in " + core.getDataDir());
        }
        name = dirs.get(0).getDirName();
    } else {
        //"snapshot." is prefixed by snapshooter
        name = "snapshot." + name;
    }
    RestoreCore restoreCore = new RestoreCore(repo, core, locationUri, name);
    try {
        MDC.put("RestoreCore.core", core.getName());
        MDC.put("RestoreCore.backupLocation", location);
        MDC.put("RestoreCore.backupName", name);
        restoreFuture = restoreExecutor.submit(restoreCore);
        currentRestoreName = name;
    } finally {
        MDC.remove("RestoreCore.core");
        MDC.remove("RestoreCore.backupLocation");
        MDC.remove("RestoreCore.backupName");
    }
}

Example 2 with BackupRepository

use of org.apache.solr.core.backup.repository.BackupRepository in project lucene-solr by apache.

the class BackupCoreOp method execute.

@Override
public void execute(CoreAdminHandler.CallInfo it) throws Exception {
    ZkController zkController = it.handler.coreContainer.getZkController();
    if (zkController == null) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Internal SolrCloud API");
    }
    final SolrParams params = it.req.getParams();
    String cname = params.get(CoreAdminParams.CORE);
    if (cname == null) {
        throw new IllegalArgumentException(CoreAdminParams.CORE + " is required");
    }
    String name = params.get(NAME);
    if (name == null) {
        throw new IllegalArgumentException(CoreAdminParams.NAME + " is required");
    }
    String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY);
    BackupRepository repository = it.handler.coreContainer.newBackupRepository(Optional.ofNullable(repoName));
    String location = repository.getBackupLocation(params.get(CoreAdminParams.BACKUP_LOCATION));
    if (location == null) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'location' is not specified as a query" + " parameter or as a default repository property");
    }
    // An optional parameter to describe the snapshot to be backed-up. If this
    // parameter is not supplied, the latest index commit is backed-up.
    String commitName = params.get(CoreAdminParams.COMMIT_NAME);
    URI locationUri = repository.createURI(location);
    try (SolrCore core = it.handler.coreContainer.getCore(cname)) {
        SnapShooter snapShooter = new SnapShooter(repository, core, locationUri, name, commitName);
        //  file system. Otherwise, perhaps the FS location isn't shared -- we want an error.
        if (!snapShooter.getBackupRepository().exists(snapShooter.getLocation())) {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Directory to contain snapshots doesn't exist: " + snapShooter.getLocation());
        }
        snapShooter.validateCreateSnapshot();
        snapShooter.createSnapshot();
    } catch (Exception e) {
        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Failed to backup core=" + cname + " because " + e, e);
    }
}

Example 3 with BackupRepository

use of org.apache.solr.core.backup.repository.BackupRepository in project lucene-solr by apache.

the class BackupCmd method call.

@Override
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
    String collectionName = message.getStr(COLLECTION_PROP);
    String backupName = message.getStr(NAME);
    String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY);
    Instant startTime = Instant.now();
    CoreContainer cc = ocmh.overseer.getZkController().getCoreContainer();
    BackupRepository repository = cc.newBackupRepository(Optional.ofNullable(repo));
    BackupManager backupMgr = new BackupManager(repository, ocmh.zkStateReader);
    // Backup location
    URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION));
    URI backupPath = repository.resolve(location, backupName);
    //Validating if the directory already exists.
    if (repository.exists(backupPath)) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The backup directory already exists: " + backupPath);
    }
    // Create a directory to store backup details.
    repository.createDirectory(backupPath);
    String strategy = message.getStr(CollectionAdminParams.INDEX_BACKUP_STRATEGY, CollectionAdminParams.COPY_FILES_STRATEGY);
    switch(strategy) {
        case CollectionAdminParams.COPY_FILES_STRATEGY:
            {
                copyIndexFiles(backupPath, message, results);
                break;
            }
        case CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY:
            {
                break;
            }
    }
    log.info("Starting to backup ZK data for backupName={}", backupName);
    //Download the configs
    String configName = ocmh.zkStateReader.readConfigName(collectionName);
    backupMgr.downloadConfigDir(location, backupName, configName);
    //Save the collection's state. Can be part of the monolithic clusterstate.json or a individual state.json
    //Since we don't want to distinguish we extract the state and back it up as a separate json
    DocCollection collectionState = ocmh.zkStateReader.getClusterState().getCollection(collectionName);
    backupMgr.writeCollectionState(location, backupName, collectionName, collectionState);
    Properties properties = new Properties();
    properties.put(BackupManager.BACKUP_NAME_PROP, backupName);
    properties.put(BackupManager.COLLECTION_NAME_PROP, collectionName);
    properties.put(COLL_CONF, configName);
    properties.put(BackupManager.START_TIME_PROP, startTime.toString());
    properties.put(BackupManager.INDEX_VERSION_PROP, Version.LATEST.toString());
    //TODO: Add MD5 of the configset. If during restore the same name configset exists then we can compare checksums to see if they are the same.
    //if they are not the same then we can throw an error or have an 'overwriteConfig' flag
    //TODO save numDocs for the shardLeader. We can use it to sanity check the restore.
    backupMgr.writeBackupProperties(location, backupName, properties);
    log.info("Completed backing up ZK data for backupName={}", backupName);
}

Example 4 with BackupRepository

use of org.apache.solr.core.backup.repository.BackupRepository in project lucene-solr by apache.

the class TestBackupRepositoryFactory method testRepositoryConfig.

@Test
public void testRepositoryConfig() {
    PluginInfo[] plugins = new PluginInfo[1];
    {
        Map<String, Object> attrs = new HashMap<>();
        attrs.put(CoreAdminParams.NAME, "repo1");
        attrs.put(FieldType.CLASS_NAME, LocalFileSystemRepository.class.getName());
        attrs.put("default", "true");
        attrs.put("location", "/tmp");
        plugins[0] = new PluginInfo("repository", attrs);
    }
    BackupRepositoryFactory f = new BackupRepositoryFactory(plugins);
    {
        BackupRepository repo = f.newInstance(loader);
        assertTrue(repo instanceof LocalFileSystemRepository);
        assertEquals("/tmp", repo.getConfigProperty("location"));
    }
    {
        BackupRepository repo = f.newInstance(loader, "repo1");
        assertTrue(repo instanceof LocalFileSystemRepository);
        assertEquals("/tmp", repo.getConfigProperty("location"));
    }
}

Example 5 with BackupRepository

use of org.apache.solr.core.backup.repository.BackupRepository in project lucene-solr by apache.

the class ReplicationHandler method doSnapShoot.

private void doSnapShoot(SolrParams params, SolrQueryResponse rsp, SolrQueryRequest req) {
    try {
        int numberToKeep = params.getInt(NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM, 0);
        if (numberToKeep > 0 && numberBackupsToKeep > 0) {
            throw new SolrException(ErrorCode.BAD_REQUEST, "Cannot use " + NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM + " if " + NUMBER_BACKUPS_TO_KEEP_INIT_PARAM + " was specified in the configuration.");
        }
        numberToKeep = Math.max(numberToKeep, numberBackupsToKeep);
        if (numberToKeep < 1) {
            numberToKeep = Integer.MAX_VALUE;
        }
        IndexCommit indexCommit = null;
        String commitName = params.get(CoreAdminParams.COMMIT_NAME);
        if (commitName != null) {
            SolrSnapshotMetaDataManager snapshotMgr = core.getSnapshotMetaDataManager();
            Optional<IndexCommit> commit = snapshotMgr.getIndexCommitByName(commitName);
            if (commit.isPresent()) {
                indexCommit = commit.get();
            } else {
                throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to find an index commit with name " + commitName + " for core " + core.getName());
            }
        } else {
            IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy();
            indexCommit = delPolicy.getLatestCommit();
            if (indexCommit == null) {
                indexCommit = req.getSearcher().getIndexReader().getIndexCommit();
            }
        }
        String location = params.get(CoreAdminParams.BACKUP_LOCATION);
        String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY);
        CoreContainer cc = core.getCoreContainer();
        BackupRepository repo = null;
        if (repoName != null) {
            repo = cc.newBackupRepository(Optional.of(repoName));
            location = repo.getBackupLocation(location);
            if (location == null) {
                throw new IllegalArgumentException("location is required");
            }
        } else {
            repo = new LocalFileSystemRepository();
            if (location == null) {
                location = core.getDataDir();
            } else {
                location = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString();
            }
        }
        // small race here before the commit point is saved
        URI locationUri = repo.createURI(location);
        SnapShooter snapShooter = new SnapShooter(repo, core, locationUri, params.get(NAME), commitName);
        snapShooter.validateCreateSnapshot();
        snapShooter.createSnapAsync(indexCommit, numberToKeep, (nl) -> snapShootDetails = nl);
    } catch (Exception e) {
        LOG.warn("Exception during creating a snapshot", e);
        rsp.add("exception", e);
    }
}