Search in sources :

Example 1 with CreateSnapshot

use of org.apache.solr.client.solrj.request.CoreAdminRequest.CreateSnapshot in project lucene-solr by apache.

the class TestSolrCoreSnapshots method testIndexOptimization.

@Test
public void testIndexOptimization() throws Exception {
    CloudSolrClient solrClient = cluster.getSolrClient();
    String collectionName = "SolrCoreSnapshots_IndexOptimization";
    CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1);
    create.process(solrClient);
    int nDocs = BackupRestoreUtils.indexDocs(cluster.getSolrClient(), collectionName, docsSeed);
    DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
    assertEquals(1, collectionState.getActiveSlices().size());
    Slice shard = collectionState.getActiveSlices().iterator().next();
    assertEquals(1, shard.getReplicas().size());
    Replica replica = shard.getReplicas().iterator().next();
    String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP);
    String commitName = TestUtil.randomSimpleString(random(), 1, 5);
    try (SolrClient adminClient = getHttpSolrClient(cluster.getJettySolrRunners().get(0).getBaseUrl().toString());
        SolrClient masterClient = getHttpSolrClient(replica.getCoreUrl())) {
        SnapshotMetaData metaData = createSnapshot(adminClient, coreName, commitName);
        int numTests = nDocs > 0 ? TestUtil.nextInt(random(), 1, 5) : 1;
        for (int attempt = 0; attempt < numTests; attempt++) {
            //Modify existing index before we call optimize.
            if (nDocs > 0) {
                //Delete a few docs
                int numDeletes = TestUtil.nextInt(random(), 1, nDocs);
                for (int i = 0; i < numDeletes; i++) {
                    masterClient.deleteByQuery("id:" + i);
                }
                //Add a few more
                int moreAdds = TestUtil.nextInt(random(), 1, 100);
                for (int i = 0; i < moreAdds; i++) {
                    SolrInputDocument doc = new SolrInputDocument();
                    doc.addField("id", i + nDocs);
                    doc.addField("name", "name = " + (i + nDocs));
                    masterClient.add(doc);
                }
                masterClient.commit();
            }
        }
        // Before invoking optimize command, verify that the index directory contains multiple commits (including the one we snapshotted earlier).
        {
            Collection<IndexCommit> commits = listCommits(metaData.getIndexDirPath());
            // Verify that multiple index commits are stored in this directory.
            assertTrue(commits.size() > 0);
            // Verify that the snapshot commit is present in this directory.
            assertTrue(commits.stream().filter(x -> x.getGeneration() == metaData.getGenerationNumber()).findFirst().isPresent());
        }
        // Optimize the index.
        masterClient.optimize(true, true, 1);
        // After invoking optimize command, verify that the index directory contains multiple commits (including the one we snapshotted earlier).
        {
            List<IndexCommit> commits = listCommits(metaData.getIndexDirPath());
            // Verify that multiple index commits are stored in this directory.
            assertTrue(commits.size() > 1);
            // Verify that the snapshot commit is present in this directory.
            assertTrue(commits.stream().filter(x -> x.getGeneration() == metaData.getGenerationNumber()).findFirst().isPresent());
        }
        // Delete the snapshot
        deleteSnapshot(adminClient, coreName, metaData.getName());
        // Add few documents. Without this the optimize command below does not take effect.
        {
            int moreAdds = TestUtil.nextInt(random(), 1, 100);
            for (int i = 0; i < moreAdds; i++) {
                SolrInputDocument doc = new SolrInputDocument();
                doc.addField("id", i + nDocs);
                doc.addField("name", "name = " + (i + nDocs));
                masterClient.add(doc);
            }
            masterClient.commit();
        }
        // Optimize the index.
        masterClient.optimize(true, true, 1);
        // Verify that the index directory contains only 1 index commit (which is not the same as the snapshotted commit).
        Collection<IndexCommit> commits = listCommits(metaData.getIndexDirPath());
        assertTrue(commits.size() == 1);
        assertFalse(commits.stream().filter(x -> x.getGeneration() == metaData.getGenerationNumber()).findFirst().isPresent());
    }
}
Also used : IndexCommit(org.apache.lucene.index.IndexCommit) IndexNotFoundException(org.apache.lucene.index.IndexNotFoundException) BeforeClass(org.junit.BeforeClass) Slow(org.apache.lucene.util.LuceneTestCase.Slow) DocCollection(org.apache.solr.common.cloud.DocCollection) TestUtil(org.apache.lucene.util.TestUtil) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) SolrTestCaseJ4(org.apache.solr.SolrTestCaseJ4) ArrayList(java.util.ArrayList) BackupRestoreUtils(org.apache.solr.handler.BackupRestoreUtils) BASE_URL_PROP(org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP) Map(java.util.Map) DeleteSnapshot(org.apache.solr.client.solrj.request.CoreAdminRequest.DeleteSnapshot) CloudSolrClient(org.apache.solr.client.solrj.impl.CloudSolrClient) SimpleFSDirectory(org.apache.lucene.store.SimpleFSDirectory) ZkStateReader(org.apache.solr.common.cloud.ZkStateReader) AfterClass(org.junit.AfterClass) Slice(org.apache.solr.common.cloud.Slice) Logger(org.slf4j.Logger) MethodHandles(java.lang.invoke.MethodHandles) Collection(java.util.Collection) DirectoryReader(org.apache.lucene.index.DirectoryReader) SolrCloudTestCase(org.apache.solr.cloud.SolrCloudTestCase) Test(org.junit.Test) CreateSnapshot(org.apache.solr.client.solrj.request.CoreAdminRequest.CreateSnapshot) Replica(org.apache.solr.common.cloud.Replica) NamedList(org.apache.solr.common.util.NamedList) SolrClient(org.apache.solr.client.solrj.SolrClient) SnapshotMetaData(org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData) List(java.util.List) ListSnapshots(org.apache.solr.client.solrj.request.CoreAdminRequest.ListSnapshots) Paths(java.nio.file.Paths) CoreAdminAction(org.apache.solr.common.params.CoreAdminParams.CoreAdminAction) Optional(java.util.Optional) Collections(java.util.Collections) CollectionAdminRequest(org.apache.solr.client.solrj.request.CollectionAdminRequest) SolrInputDocument(org.apache.solr.common.SolrInputDocument) CollectionAdminRequest(org.apache.solr.client.solrj.request.CollectionAdminRequest) SnapshotMetaData(org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData) Replica(org.apache.solr.common.cloud.Replica) IndexCommit(org.apache.lucene.index.IndexCommit) CloudSolrClient(org.apache.solr.client.solrj.impl.CloudSolrClient) SolrInputDocument(org.apache.solr.common.SolrInputDocument) CloudSolrClient(org.apache.solr.client.solrj.impl.CloudSolrClient) SolrClient(org.apache.solr.client.solrj.SolrClient) Slice(org.apache.solr.common.cloud.Slice) DocCollection(org.apache.solr.common.cloud.DocCollection) Collection(java.util.Collection) ArrayList(java.util.ArrayList) NamedList(org.apache.solr.common.util.NamedList) List(java.util.List) DocCollection(org.apache.solr.common.cloud.DocCollection) Test(org.junit.Test)

Example 2 with CreateSnapshot

use of org.apache.solr.client.solrj.request.CoreAdminRequest.CreateSnapshot in project lucene-solr by apache.

the class TestSolrCoreSnapshots method createSnapshot.

private SnapshotMetaData createSnapshot(SolrClient adminClient, String coreName, String commitName) throws Exception {
    CreateSnapshot req = new CreateSnapshot(commitName);
    req.setCoreName(coreName);
    adminClient.request(req);
    Collection<SnapshotMetaData> snapshots = listSnapshots(adminClient, coreName);
    Optional<SnapshotMetaData> metaData = snapshots.stream().filter(x -> commitName.equals(x.getName())).findFirst();
    assertTrue(metaData.isPresent());
    return metaData.get();
}
Also used : IndexCommit(org.apache.lucene.index.IndexCommit) IndexNotFoundException(org.apache.lucene.index.IndexNotFoundException) BeforeClass(org.junit.BeforeClass) Slow(org.apache.lucene.util.LuceneTestCase.Slow) DocCollection(org.apache.solr.common.cloud.DocCollection) TestUtil(org.apache.lucene.util.TestUtil) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) SolrTestCaseJ4(org.apache.solr.SolrTestCaseJ4) ArrayList(java.util.ArrayList) BackupRestoreUtils(org.apache.solr.handler.BackupRestoreUtils) BASE_URL_PROP(org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP) Map(java.util.Map) DeleteSnapshot(org.apache.solr.client.solrj.request.CoreAdminRequest.DeleteSnapshot) CloudSolrClient(org.apache.solr.client.solrj.impl.CloudSolrClient) SimpleFSDirectory(org.apache.lucene.store.SimpleFSDirectory) ZkStateReader(org.apache.solr.common.cloud.ZkStateReader) AfterClass(org.junit.AfterClass) Slice(org.apache.solr.common.cloud.Slice) Logger(org.slf4j.Logger) MethodHandles(java.lang.invoke.MethodHandles) Collection(java.util.Collection) DirectoryReader(org.apache.lucene.index.DirectoryReader) SolrCloudTestCase(org.apache.solr.cloud.SolrCloudTestCase) Test(org.junit.Test) CreateSnapshot(org.apache.solr.client.solrj.request.CoreAdminRequest.CreateSnapshot) Replica(org.apache.solr.common.cloud.Replica) NamedList(org.apache.solr.common.util.NamedList) SolrClient(org.apache.solr.client.solrj.SolrClient) SnapshotMetaData(org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData) List(java.util.List) ListSnapshots(org.apache.solr.client.solrj.request.CoreAdminRequest.ListSnapshots) Paths(java.nio.file.Paths) CoreAdminAction(org.apache.solr.common.params.CoreAdminParams.CoreAdminAction) Optional(java.util.Optional) Collections(java.util.Collections) CollectionAdminRequest(org.apache.solr.client.solrj.request.CollectionAdminRequest) SolrInputDocument(org.apache.solr.common.SolrInputDocument) SnapshotMetaData(org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData) CreateSnapshot(org.apache.solr.client.solrj.request.CoreAdminRequest.CreateSnapshot)

Aggregations

MethodHandles (java.lang.invoke.MethodHandles)2 Paths (java.nio.file.Paths)2 ArrayList (java.util.ArrayList)2 Collection (java.util.Collection)2 Collections (java.util.Collections)2 HashMap (java.util.HashMap)2 List (java.util.List)2 Map (java.util.Map)2 Optional (java.util.Optional)2 DirectoryReader (org.apache.lucene.index.DirectoryReader)2 IndexCommit (org.apache.lucene.index.IndexCommit)2 IndexNotFoundException (org.apache.lucene.index.IndexNotFoundException)2 SimpleFSDirectory (org.apache.lucene.store.SimpleFSDirectory)2 Slow (org.apache.lucene.util.LuceneTestCase.Slow)2 TestUtil (org.apache.lucene.util.TestUtil)2 SolrTestCaseJ4 (org.apache.solr.SolrTestCaseJ4)2 SolrClient (org.apache.solr.client.solrj.SolrClient)2 CloudSolrClient (org.apache.solr.client.solrj.impl.CloudSolrClient)2 CollectionAdminRequest (org.apache.solr.client.solrj.request.CollectionAdminRequest)2 CreateSnapshot (org.apache.solr.client.solrj.request.CoreAdminRequest.CreateSnapshot)2