Examples with MockCoreDescriptor org.apache.solr.util.MockCoreContainer.MockCoreDescriptor used on opensource projects

Example 1 with MockCoreDescriptor

use of org.apache.solr.util.MockCoreContainer.MockCoreDescriptor in project lucene-solr by apache.

the class HttpPartitionTest method testLeaderInitiatedRecoveryCRUD.

/**
   * Tests handling of lir state znodes.
   */
protected void testLeaderInitiatedRecoveryCRUD() throws Exception {
    String testCollectionName = "c8n_crud_1x2";
    String shardId = "shard1";
    createCollectionRetry(testCollectionName, 1, 2, 1);
    cloudClient.setDefaultCollection(testCollectionName);
    Replica leader = cloudClient.getZkStateReader().getLeaderRetry(testCollectionName, shardId);
    JettySolrRunner leaderJetty = getJettyOnPort(getReplicaPort(leader));
    CoreContainer cores = leaderJetty.getCoreContainer();
    ZkController zkController = cores.getZkController();
    assertNotNull("ZkController is null", zkController);
    Replica notLeader = ensureAllReplicasAreActive(testCollectionName, shardId, 1, 2, maxWaitSecsToSeeAllActive).get(0);
    ZkCoreNodeProps replicaCoreNodeProps = new ZkCoreNodeProps(notLeader);
    String replicaUrl = replicaCoreNodeProps.getCoreUrl();
    MockCoreDescriptor cd = new MockCoreDescriptor() {

        public CloudDescriptor getCloudDescriptor() {
            return new CloudDescriptor(leader.getStr(ZkStateReader.CORE_NAME_PROP), new Properties(), this) {

                @Override
                public String getCoreNodeName() {
                    return leader.getName();
                }

                @Override
                public boolean isLeader() {
                    return true;
                }
            };
        }
    };
    zkController.updateLeaderInitiatedRecoveryState(testCollectionName, shardId, notLeader.getName(), Replica.State.DOWN, cd, true);
    Map<String, Object> lirStateMap = zkController.getLeaderInitiatedRecoveryStateObject(testCollectionName, shardId, notLeader.getName());
    assertNotNull(lirStateMap);
    assertSame(Replica.State.DOWN, Replica.State.getState((String) lirStateMap.get(ZkStateReader.STATE_PROP)));
    // test old non-json format handling
    SolrZkClient zkClient = zkController.getZkClient();
    String znodePath = zkController.getLeaderInitiatedRecoveryZnodePath(testCollectionName, shardId, notLeader.getName());
    zkClient.setData(znodePath, "down".getBytes(StandardCharsets.UTF_8), true);
    lirStateMap = zkController.getLeaderInitiatedRecoveryStateObject(testCollectionName, shardId, notLeader.getName());
    assertNotNull(lirStateMap);
    assertSame(Replica.State.DOWN, Replica.State.getState((String) lirStateMap.get(ZkStateReader.STATE_PROP)));
    zkClient.delete(znodePath, -1, false);
    // try to clean up
    attemptCollectionDelete(cloudClient, testCollectionName);
}

Example 2 with MockCoreDescriptor

use of org.apache.solr.util.MockCoreContainer.MockCoreDescriptor in project lucene-solr by apache.

the class HdfsDirectoryFactoryTest method testCleanupOldIndexDirectories.

@Test
public void testCleanupOldIndexDirectories() throws Exception {
    HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory();
    System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1");
    hdfsFactory.init(new NamedList<>());
    String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
    assertTrue(dataHome.endsWith("/solr1/mock/data"));
    System.clearProperty("solr.hdfs.home");
    FileSystem hdfs = dfsCluster.getFileSystem();
    org.apache.hadoop.fs.Path dataHomePath = new org.apache.hadoop.fs.Path(dataHome);
    org.apache.hadoop.fs.Path currentIndexDirPath = new org.apache.hadoop.fs.Path(dataHomePath, "index");
    assertTrue(!hdfs.isDirectory(currentIndexDirPath));
    hdfs.mkdirs(currentIndexDirPath);
    assertTrue(hdfs.isDirectory(currentIndexDirPath));
    String timestamp1 = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date());
    org.apache.hadoop.fs.Path oldIndexDirPath = new org.apache.hadoop.fs.Path(dataHomePath, "index." + timestamp1);
    assertTrue(!hdfs.isDirectory(oldIndexDirPath));
    hdfs.mkdirs(oldIndexDirPath);
    assertTrue(hdfs.isDirectory(oldIndexDirPath));
    hdfsFactory.cleanupOldIndexDirectories(dataHomePath.toString(), currentIndexDirPath.toString(), false);
    assertTrue(hdfs.isDirectory(currentIndexDirPath));
    assertTrue(!hdfs.isDirectory(oldIndexDirPath));
}

Example 3 with MockCoreDescriptor

use of org.apache.solr.util.MockCoreContainer.MockCoreDescriptor in project lucene-solr by apache.

the class HdfsDirectoryFactoryTest method testInitArgsOrSysPropConfig.

@Test
public void testInitArgsOrSysPropConfig() throws Exception {
    HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory();
    // test sys prop config
    System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1");
    hdfsFactory.init(new NamedList<>());
    String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
    assertTrue(dataHome.endsWith("/solr1/mock/data"));
    System.clearProperty("solr.hdfs.home");
    // test init args config
    NamedList<Object> nl = new NamedList<>();
    nl.add("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr2");
    hdfsFactory.init(nl);
    dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
    assertTrue(dataHome.endsWith("/solr2/mock/data"));
    // test sys prop and init args config - init args wins
    System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1");
    hdfsFactory.init(nl);
    dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
    assertTrue(dataHome.endsWith("/solr2/mock/data"));
    System.clearProperty("solr.hdfs.home");
    // set conf dir by sys prop
    Path confDir = createTempDir();
    System.setProperty(HdfsDirectoryFactory.CONFIG_DIRECTORY, confDir.toString());
    Directory dir = hdfsFactory.create(HdfsTestUtil.getURI(dfsCluster) + "/solr", NoLockFactory.INSTANCE, DirContext.DEFAULT);
    try {
        assertEquals(confDir.toString(), hdfsFactory.getConfDir());
    } finally {
        dir.close();
    }
    // check bool and int getConf impls
    nl = new NamedList<>();
    nl.add(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 4);
    System.setProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, "3");
    nl.add(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, true);
    System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "false");
    hdfsFactory.init(nl);
    assertEquals(4, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
    assertEquals(true, hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
    nl = new NamedList<>();
    hdfsFactory.init(nl);
    System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "true");
    assertEquals(3, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
    assertEquals(true, hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
    System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
    System.clearProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED);
    assertEquals(0, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
    assertEquals(false, hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
    hdfsFactory.close();
}

Example 4 with MockCoreDescriptor

use of org.apache.solr.util.MockCoreContainer.MockCoreDescriptor in project lucene-solr by apache.

the class TestLeaderInitiatedRecoveryThread method testPublishDownState.

public void testPublishDownState() throws Exception {
    waitForRecoveriesToFinish(true);
    final String leaderCoreNodeName = shardToLeaderJetty.get(SHARD1).coreNodeName;
    final CloudJettyRunner leaderRunner = shardToLeaderJetty.get(SHARD1);
    CoreContainer coreContainer = leaderRunner.jetty.getCoreContainer();
    ZkController zkController = coreContainer.getZkController();
    CloudJettyRunner notLeader = null;
    for (CloudJettyRunner cloudJettyRunner : shardToJetty.get(SHARD1)) {
        if (cloudJettyRunner != leaderRunner) {
            notLeader = cloudJettyRunner;
            break;
        }
    }
    assertNotNull(notLeader);
    Replica replica = cloudClient.getZkStateReader().getClusterState().getReplica(DEFAULT_COLLECTION, notLeader.coreNodeName);
    ZkCoreNodeProps replicaCoreNodeProps = new ZkCoreNodeProps(replica);
    MockCoreDescriptor cd = new MockCoreDescriptor() {

        public CloudDescriptor getCloudDescriptor() {
            return new CloudDescriptor(shardToLeaderJetty.get(SHARD1).info.getStr(ZkStateReader.CORE_NAME_PROP), new Properties(), this) {

                @Override
                public String getCoreNodeName() {
                    return shardToLeaderJetty.get(SHARD1).info.getStr(ZkStateReader.CORE_NODE_NAME_PROP);
                }

                @Override
                public boolean isLeader() {
                    return true;
                }
            };
        }
    };
    /*
     1. Test that publishDownState throws exception when zkController.isReplicaInRecoveryHandling == false
      */
    try {
        LeaderInitiatedRecoveryThread thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer, DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, cd);
        assertFalse(zkController.isReplicaInRecoveryHandling(replicaCoreNodeProps.getCoreUrl()));
        thread.run();
        fail("publishDownState should not have succeeded because replica url is not marked in leader initiated recovery in ZkController");
    } catch (SolrException e) {
        assertTrue(e.code() == SolrException.ErrorCode.INVALID_STATE.code);
    }
    /*
     2. Test that a non-live replica cannot be put into LIR or down state
      */
    LeaderInitiatedRecoveryThread thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer, DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, cd);
    // kill the replica
    int children = cloudClient.getZkStateReader().getZkClient().getChildren("/live_nodes", null, true).size();
    ChaosMonkey.stop(notLeader.jetty);
    TimeOut timeOut = new TimeOut(60, TimeUnit.SECONDS);
    while (!timeOut.hasTimedOut()) {
        if (children > cloudClient.getZkStateReader().getZkClient().getChildren("/live_nodes", null, true).size()) {
            break;
        }
        Thread.sleep(500);
    }
    assertTrue(children > cloudClient.getZkStateReader().getZkClient().getChildren("/live_nodes", null, true).size());
    int cversion = getOverseerCversion();
    // Thread should not publish LIR and down state for node which is not live, regardless of whether forcePublish is true or false
    assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), false));
    // lets assert that we did not publish anything to overseer queue, simplest way is to assert that cversion of overseer queue zk node is still the same
    assertEquals(cversion, getOverseerCversion());
    assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), true));
    // lets assert that we did not publish anything to overseer queue
    assertEquals(cversion, getOverseerCversion());
    /*
    3. Test that if ZK connection loss then thread should not attempt to publish down state even if forcePublish=true
     */
    ChaosMonkey.start(notLeader.jetty);
    waitForRecoveriesToFinish(true);
    thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer, DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, cd) {

        @Override
        protected void updateLIRState(String replicaCoreNodeName) {
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "", new KeeperException.ConnectionLossException());
        }
    };
    assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), false));
    assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), true));
    assertNull(zkController.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
    /*
     4. Test that if ZK connection loss or session expired then thread should not attempt to publish down state even if forcePublish=true
      */
    thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer, DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, cd) {

        @Override
        protected void updateLIRState(String replicaCoreNodeName) {
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "", new KeeperException.SessionExpiredException());
        }
    };
    assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), false));
    assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), true));
    assertNull(zkController.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
    /*
     5. Test that any exception other then ZK connection loss or session expired should publish down state only if forcePublish=true
      */
    thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer, DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, cd) {

        @Override
        protected void updateLIRState(String replicaCoreNodeName) {
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "bogus exception");
        }
    };
    // the following should return true because regardless of the bogus exception in setting LIR state, we still want recovery commands to be sent,
    // however the following will not publish a down state
    cversion = getOverseerCversion();
    assertTrue(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), false));
    // lets assert that we did not publish anything to overseer queue, simplest way is to assert that cversion of overseer queue zk node is still the same
    assertEquals(cversion, getOverseerCversion());
    assertTrue(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), true));
    // this should have published a down state so assert that cversion has incremented
    assertTrue(getOverseerCversion() > cversion);
    timeOut = new TimeOut(30, TimeUnit.SECONDS);
    while (!timeOut.hasTimedOut()) {
        Replica r = cloudClient.getZkStateReader().getClusterState().getReplica(DEFAULT_COLLECTION, replica.getName());
        if (r.getState() == Replica.State.DOWN) {
            break;
        }
        Thread.sleep(500);
    }
    assertNull(zkController.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
    assertEquals(Replica.State.DOWN, cloudClient.getZkStateReader().getClusterState().getReplica(DEFAULT_COLLECTION, replica.getName()).getState());
    /*
    6. Test that non-leader cannot set LIR nodes
     */
    coreContainer = notLeader.jetty.getCoreContainer();
    zkController = coreContainer.getZkController();
    thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer, DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, coreContainer.getCores().iterator().next().getCoreDescriptor()) {

        @Override
        protected void updateLIRState(String replicaCoreNodeName) {
            try {
                super.updateLIRState(replicaCoreNodeName);
            } catch (Exception e) {
                assertTrue(e instanceof ZkController.NotLeaderException);
                throw e;
            }
        }
    };
    cversion = getOverseerCversion();
    assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), false));
    assertEquals(cversion, getOverseerCversion());
    /*
     7. assert that we can write a LIR state if everything else is fine
      */
    // reset the zkcontroller to the one from the leader
    coreContainer = leaderRunner.jetty.getCoreContainer();
    zkController = coreContainer.getZkController();
    thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer, DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, coreContainer.getCores().iterator().next().getCoreDescriptor());
    thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), false);
    timeOut = new TimeOut(30, TimeUnit.SECONDS);
    while (!timeOut.hasTimedOut()) {
        Replica.State state = zkController.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName());
        if (state == Replica.State.DOWN) {
            break;
        }
        Thread.sleep(500);
    }
    assertNotNull(zkController.getLeaderInitiatedRecoveryStateObject(DEFAULT_COLLECTION, SHARD1, replica.getName()));
    assertEquals(Replica.State.DOWN, zkController.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
/*
    7. Test that
     */
}