Examples with SnapshotMetaData org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData used on opensource projects

Example 6 with SnapshotMetaData

use of org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData in project lucene-solr by apache.

the class SolrSnapshotManager method deleteNonSnapshotIndexFiles.

/**
   * This method deletes index files not associated with the specified <code>snapshots</code>.
   *
   * @param core The Solr core
   * @param dir The index directory storing the snapshot.
   * @param snapshots The snapshots to be preserved.
   * @throws IOException in case of I/O errors.
   */
public static void deleteNonSnapshotIndexFiles(SolrCore core, Directory dir, Collection<SnapshotMetaData> snapshots) throws IOException {
    final Set<Long> genNumbers = new HashSet<>();
    for (SnapshotMetaData m : snapshots) {
        genNumbers.add(m.getGenerationNumber());
    }
    deleteSnapshotIndexFiles(core, dir, new IndexDeletionPolicy() {

        @Override
        public void onInit(List<? extends IndexCommit> commits) throws IOException {
            for (IndexCommit ic : commits) {
                if (!genNumbers.contains(ic.getGeneration())) {
                    log.info("Deleting non-snapshotted index commit with generation {}", ic.getGeneration());
                    ic.delete();
                }
            }
        }

        @Override
        public void onCommit(List<? extends IndexCommit> commits) throws IOException {
        }
    });
}

Example 7 with SnapshotMetaData

use of org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData in project lucene-solr by apache.

the class TestSolrCoreSnapshots method createSnapshot.

private SnapshotMetaData createSnapshot(SolrClient adminClient, String coreName, String commitName) throws Exception {
    CreateSnapshot req = new CreateSnapshot(commitName);
    req.setCoreName(coreName);
    adminClient.request(req);
    Collection<SnapshotMetaData> snapshots = listSnapshots(adminClient, coreName);
    Optional<SnapshotMetaData> metaData = snapshots.stream().filter(x -> commitName.equals(x.getName())).findFirst();
    assertTrue(metaData.isPresent());
    return metaData.get();
}

Example 8 with SnapshotMetaData

use of org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData in project lucene-solr by apache.

the class TestSolrCoreSnapshots method testBackupRestore.

@Test
public void testBackupRestore() throws Exception {
    CloudSolrClient solrClient = cluster.getSolrClient();
    String collectionName = "SolrCoreSnapshots";
    CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1);
    create.process(solrClient);
    String location = createTempDir().toFile().getAbsolutePath();
    int nDocs = BackupRestoreUtils.indexDocs(cluster.getSolrClient(), collectionName, docsSeed);
    DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
    assertEquals(1, collectionState.getActiveSlices().size());
    Slice shard = collectionState.getActiveSlices().iterator().next();
    assertEquals(1, shard.getReplicas().size());
    Replica replica = shard.getReplicas().iterator().next();
    String replicaBaseUrl = replica.getStr(BASE_URL_PROP);
    String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP);
    String backupName = TestUtil.randomSimpleString(random(), 1, 5);
    String commitName = TestUtil.randomSimpleString(random(), 1, 5);
    String duplicateName = commitName.concat("_duplicate");
    try (SolrClient adminClient = getHttpSolrClient(cluster.getJettySolrRunners().get(0).getBaseUrl().toString());
        SolrClient masterClient = getHttpSolrClient(replica.getCoreUrl())) {
        SnapshotMetaData metaData = createSnapshot(adminClient, coreName, commitName);
        // Create another snapshot referring to the same index commit to verify the
        // reference counting implementation during snapshot deletion.
        SnapshotMetaData duplicateCommit = createSnapshot(adminClient, coreName, duplicateName);
        assertEquals(metaData.getIndexDirPath(), duplicateCommit.getIndexDirPath());
        assertEquals(metaData.getGenerationNumber(), duplicateCommit.getGenerationNumber());
        // Delete all documents
        masterClient.deleteByQuery("*:*");
        masterClient.commit();
        BackupRestoreUtils.verifyDocs(0, cluster.getSolrClient(), collectionName);
        // Verify that the index directory contains at least 2 index commits - one referred by the snapshots
        // and the other containing document deletions.
        {
            List<IndexCommit> commits = listCommits(metaData.getIndexDirPath());
            assertTrue(commits.size() >= 2);
        }
        // Backup the earlier created snapshot.
        {
            Map<String, String> params = new HashMap<>();
            params.put("name", backupName);
            params.put("commitName", commitName);
            params.put("location", location);
            BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params);
        }
        // Restore the backup
        {
            Map<String, String> params = new HashMap<>();
            params.put("name", "snapshot." + backupName);
            params.put("location", location);
            BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), params);
            BackupRestoreUtils.verifyDocs(nDocs, cluster.getSolrClient(), collectionName);
        }
        // Verify that the old index directory (before restore) contains only those index commits referred by snapshots.
        // The IndexWriter (used to cleanup index files) creates an additional commit during closing. Hence we expect 2 commits (instead
        // of 1).
        {
            List<IndexCommit> commits = listCommits(metaData.getIndexDirPath());
            assertEquals(2, commits.size());
            assertEquals(metaData.getGenerationNumber(), commits.get(0).getGeneration());
        }
        // Delete first snapshot
        deleteSnapshot(adminClient, coreName, commitName);
        // Verify that corresponding index files have NOT been deleted (due to reference counting).
        assertFalse(listCommits(metaData.getIndexDirPath()).isEmpty());
        // Delete second snapshot
        deleteSnapshot(adminClient, coreName, duplicateCommit.getName());
        // Verify that corresponding index files have been deleted. Ideally this directory should
        // be removed immediately. But the current DirectoryFactory impl waits until the
        // closing the core (or the directoryFactory) for actual removal. Since the IndexWriter
        // (used to cleanup index files) creates an additional commit during closing, we expect a single
        // commit (instead of 0).
        assertEquals(1, listCommits(duplicateCommit.getIndexDirPath()).size());
    }
}

Example 9 with SnapshotMetaData

use of org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData in project lucene-solr by apache.

the class TestSolrCoreSnapshots method listSnapshots.

private Collection<SnapshotMetaData> listSnapshots(SolrClient adminClient, String coreName) throws Exception {
    ListSnapshots req = new ListSnapshots();
    req.setCoreName(coreName);
    NamedList resp = adminClient.request(req);
    assertTrue(resp.get("snapshots") instanceof NamedList);
    NamedList apiResult = (NamedList) resp.get("snapshots");
    List<SnapshotMetaData> result = new ArrayList<>(apiResult.size());
    for (int i = 0; i < apiResult.size(); i++) {
        String commitName = apiResult.getName(i);
        String indexDirPath = (String) ((NamedList) apiResult.get(commitName)).get("indexDirPath");
        long genNumber = Long.parseLong((String) ((NamedList) apiResult.get(commitName)).get("generation"));
        result.add(new SnapshotMetaData(commitName, indexDirPath, genNumber));
    }
    return result;
}

Example 10 with SnapshotMetaData

use of org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData in project lucene-solr by apache.

the class SolrCore method deleteNamedSnapshot.

/**
   * This method deletes the snapshot with the specified name. If the directory
   * storing the snapshot is not the same as the *current* core index directory,
   * then delete the files corresponding to this snapshot. Otherwise we leave the
   * index files related to snapshot as is (assuming the underlying Solr IndexDeletionPolicy
   * will clean them up appropriately).
   *
   * @param commitName The name of the snapshot to be deleted.
   * @throws IOException in case of I/O error.
   */
public void deleteNamedSnapshot(String commitName) throws IOException {
    // Note this lock is required to prevent multiple snapshot deletions from
    // opening multiple IndexWriter instances simultaneously.
    this.snapshotDelLock.lock();
    try {
        Optional<SnapshotMetaData> metadata = snapshotMgr.release(commitName);
        if (metadata.isPresent()) {
            long gen = metadata.get().getGenerationNumber();
            String indexDirPath = metadata.get().getIndexDirPath();
            if (!indexDirPath.equals(getIndexDir())) {
                Directory d = getDirectoryFactory().get(indexDirPath, DirContext.DEFAULT, "none");
                try {
                    Collection<SnapshotMetaData> snapshots = snapshotMgr.listSnapshotsInIndexDir(indexDirPath);
                    log.info("Following snapshots exist in the index directory {} : {}", indexDirPath, snapshots);
                    if (snapshots.isEmpty()) {
                        // No snapshots remain in this directory. Can be cleaned up!
                        log.info("Removing index directory {} since all named snapshots are deleted.", indexDirPath);
                        getDirectoryFactory().remove(d);
                    } else {
                        SolrSnapshotManager.deleteSnapshotIndexFiles(this, d, gen);
                    }
                } finally {
                    getDirectoryFactory().release(d);
                }
            }
        }
    } finally {
        snapshotDelLock.unlock();
    }
}