Examples with ReplicationHandler org.apache.solr.handler.ReplicationHandler used on opensource projects

Example 1 with ReplicationHandler

use of org.apache.solr.handler.ReplicationHandler in project lucene-solr by apache.

the class ReplicateFromLeader method startReplication.

/**
   * Start a replication handler thread that will periodically pull indices from the shard leader
   * @param switchTransactionLog if true, ReplicationHandler will rotate the transaction log once
   * the replication is done
   */
public void startReplication(boolean switchTransactionLog) throws InterruptedException {
    try (SolrCore core = cc.getCore(coreName)) {
        if (core == null) {
            if (cc.isShutDown()) {
                return;
            } else {
                throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "SolrCore not found:" + coreName + " in " + cc.getLoadedCoreNames());
            }
        }
        SolrConfig.UpdateHandlerInfo uinfo = core.getSolrConfig().getUpdateHandlerInfo();
        String pollIntervalStr = "00:00:03";
        if (uinfo.autoCommmitMaxTime != -1) {
            pollIntervalStr = toPollIntervalStr(uinfo.autoCommmitMaxTime / 2);
        } else if (uinfo.autoSoftCommmitMaxTime != -1) {
            pollIntervalStr = toPollIntervalStr(uinfo.autoSoftCommmitMaxTime / 2);
        }
        LOG.info("Will start replication from leader with poll interval: {}", pollIntervalStr);
        NamedList slaveConfig = new NamedList();
        slaveConfig.add("fetchFromLeader", true);
        slaveConfig.add("pollInterval", pollIntervalStr);
        NamedList replicationConfig = new NamedList();
        replicationConfig.add("slave", slaveConfig);
        String lastCommitVersion = getCommitVersion(core);
        if (lastCommitVersion != null) {
            lastVersion = Long.parseLong(lastCommitVersion);
        }
        replicationProcess = new ReplicationHandler();
        if (switchTransactionLog) {
            replicationProcess.setPollListener((solrCore, pollSuccess) -> {
                if (pollSuccess) {
                    String commitVersion = getCommitVersion(core);
                    if (commitVersion == null)
                        return;
                    if (Long.parseLong(commitVersion) == lastVersion)
                        return;
                    UpdateLog updateLog = solrCore.getUpdateHandler().getUpdateLog();
                    SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams());
                    CommitUpdateCommand cuc = new CommitUpdateCommand(req, false);
                    cuc.setVersion(Long.parseLong(commitVersion));
                    updateLog.copyOverOldUpdates(cuc);
                    lastVersion = Long.parseLong(commitVersion);
                }
            });
        }
        replicationProcess.init(replicationConfig);
        replicationProcess.inform(core);
    }
}

Example 2 with ReplicationHandler

use of org.apache.solr.handler.ReplicationHandler in project lucene-solr by apache.

the class TestRuleBasedAuthorizationPlugin method testBasicPermissions.

public void testBasicPermissions() {
    int STATUS_OK = 200;
    int FORBIDDEN = 403;
    int PROMPT_FOR_CREDENTIALS = 401;
    checkRules(makeMap("resource", "/update/json/docs", "httpMethod", "POST", "userPrincipal", "unknownuser", "collectionRequests", "freeforall", "handler", new UpdateRequestHandler()), STATUS_OK);
    checkRules(makeMap("resource", "/update/json/docs", "httpMethod", "POST", "userPrincipal", "tim", "collectionRequests", "mycoll", "handler", new UpdateRequestHandler()), STATUS_OK);
    checkRules(makeMap("resource", "/update/json/docs", "httpMethod", "POST", "collectionRequests", "mycoll", "handler", new UpdateRequestHandler()), PROMPT_FOR_CREDENTIALS);
    checkRules(makeMap("resource", "/schema", "userPrincipal", "somebody", "collectionRequests", "mycoll", "httpMethod", "POST", "handler", new SchemaHandler()), FORBIDDEN);
    checkRules(makeMap("resource", "/schema", "userPrincipal", "somebody", "collectionRequests", "mycoll", "httpMethod", "GET", "handler", new SchemaHandler()), STATUS_OK);
    checkRules(makeMap("resource", "/schema/fields", "userPrincipal", "somebody", "collectionRequests", "mycoll", "httpMethod", "GET", "handler", new SchemaHandler()), STATUS_OK);
    checkRules(makeMap("resource", "/schema", "userPrincipal", "somebody", "collectionRequests", "mycoll", "httpMethod", "POST", "handler", new SchemaHandler()), FORBIDDEN);
    checkRules(makeMap("resource", "/admin/collections", "userPrincipal", "tim", "requestType", RequestType.ADMIN, "collectionRequests", null, "httpMethod", "GET", "handler", new CollectionsHandler(), "params", new MapSolrParams(singletonMap("action", "LIST"))), STATUS_OK);
    checkRules(makeMap("resource", "/admin/collections", "userPrincipal", null, "requestType", RequestType.ADMIN, "collectionRequests", null, "httpMethod", "GET", "handler", new CollectionsHandler(), "params", new MapSolrParams(singletonMap("action", "LIST"))), STATUS_OK);
    checkRules(makeMap("resource", "/admin/collections", "userPrincipal", null, "requestType", RequestType.ADMIN, "collectionRequests", null, "handler", new CollectionsHandler(), "params", new MapSolrParams(singletonMap("action", "CREATE"))), PROMPT_FOR_CREDENTIALS);
    checkRules(makeMap("resource", "/admin/collections", "userPrincipal", null, "requestType", RequestType.ADMIN, "collectionRequests", null, "handler", new CollectionsHandler(), "params", new MapSolrParams(singletonMap("action", "RELOAD"))), PROMPT_FOR_CREDENTIALS);
    checkRules(makeMap("resource", "/admin/collections", "userPrincipal", "somebody", "requestType", RequestType.ADMIN, "collectionRequests", null, "handler", new CollectionsHandler(), "params", new MapSolrParams(singletonMap("action", "CREATE"))), FORBIDDEN);
    checkRules(makeMap("resource", "/admin/collections", "userPrincipal", "tim", "requestType", RequestType.ADMIN, "collectionRequests", null, "handler", new CollectionsHandler(), "params", new MapSolrParams(singletonMap("action", "CREATE"))), STATUS_OK);
    checkRules(makeMap("resource", "/select", "httpMethod", "GET", "handler", new SearchHandler(), "collectionRequests", singletonList(new CollectionRequest("mycoll")), "userPrincipal", "joe"), FORBIDDEN);
    Map rules = (Map) Utils.fromJSONString(permissions);
    ((Map) rules.get("user-role")).put("cio", "su");
    ((List) rules.get("permissions")).add(makeMap("name", "all", "role", "su"));
    checkRules(makeMap("resource", ReplicationHandler.PATH, "httpMethod", "POST", "userPrincipal", "tim", "handler", new ReplicationHandler(), "collectionRequests", singletonList(new CollectionRequest("mycoll"))), FORBIDDEN, rules);
    checkRules(makeMap("resource", ReplicationHandler.PATH, "httpMethod", "POST", "userPrincipal", "cio", "handler", new ReplicationHandler(), "collectionRequests", singletonList(new CollectionRequest("mycoll"))), STATUS_OK, rules);
    checkRules(makeMap("resource", "/admin/collections", "userPrincipal", "tim", "requestType", AuthorizationContext.RequestType.ADMIN, "collectionRequests", null, "handler", new CollectionsHandler(), "params", new MapSolrParams(singletonMap("action", "CREATE"))), STATUS_OK, rules);
    rules = (Map) Utils.fromJSONString(permissions);
    ((List) rules.get("permissions")).add(makeMap("name", "core-admin-edit", "role", "su"));
    ((List) rules.get("permissions")).add(makeMap("name", "core-admin-read", "role", "user"));
    ((Map) rules.get("user-role")).put("cio", "su");
    ((List) rules.get("permissions")).add(makeMap("name", "all", "role", "su"));
    permissions = Utils.toJSONString(rules);
    checkRules(makeMap("resource", "/admin/cores", "userPrincipal", null, "requestType", RequestType.ADMIN, "collectionRequests", null, "handler", new CoreAdminHandler(null), "params", new MapSolrParams(singletonMap("action", "CREATE"))), PROMPT_FOR_CREDENTIALS);
    checkRules(makeMap("resource", "/admin/cores", "userPrincipal", "joe", "requestType", RequestType.ADMIN, "collectionRequests", null, "handler", new CoreAdminHandler(null), "params", new MapSolrParams(singletonMap("action", "CREATE"))), FORBIDDEN);
    checkRules(makeMap("resource", "/admin/cores", "userPrincipal", "joe", "requestType", RequestType.ADMIN, "collectionRequests", null, "handler", new CoreAdminHandler(null), "params", new MapSolrParams(singletonMap("action", "STATUS"))), STATUS_OK);
    checkRules(makeMap("resource", "/admin/cores", "userPrincipal", "cio", "requestType", RequestType.ADMIN, "collectionRequests", null, "handler", new CoreAdminHandler(null), "params", new MapSolrParams(singletonMap("action", "CREATE"))), STATUS_OK);
    rules = (Map) Utils.fromJSONString(permissions);
    List permissions = (List) rules.get("permissions");
    //remove the 'all' permission
    permissions.remove(permissions.size() - 1);
    permissions.add(makeMap("name", "test-params", "role", "admin", "path", "/x", "params", makeMap("key", Arrays.asList("REGEX:(?i)val1", "VAL2"))));
    this.permissions = Utils.toJSONString(rules);
    checkRules(makeMap("resource", "/x", "userPrincipal", null, "requestType", RequestType.UNKNOWN, "collectionRequests", "go", "handler", new DumpRequestHandler(), "params", new MapSolrParams(singletonMap("key", "VAL1"))), PROMPT_FOR_CREDENTIALS);
    checkRules(makeMap("resource", "/x", "userPrincipal", null, "requestType", RequestType.UNKNOWN, "collectionRequests", "go", "handler", new DumpRequestHandler(), "params", new MapSolrParams(singletonMap("key", "Val1"))), PROMPT_FOR_CREDENTIALS);
    checkRules(makeMap("resource", "/x", "userPrincipal", null, "requestType", RequestType.UNKNOWN, "collectionRequests", "go", "handler", new DumpRequestHandler(), "params", new MapSolrParams(singletonMap("key", "Val1"))), PROMPT_FOR_CREDENTIALS);
    checkRules(makeMap("resource", "/x", "userPrincipal", "joe", "requestType", RequestType.UNKNOWN, "collectionRequests", "go", "handler", new DumpRequestHandler(), "params", new MapSolrParams(singletonMap("key", "Val1"))), FORBIDDEN);
    checkRules(makeMap("resource", "/x", "userPrincipal", "joe", "requestType", RequestType.UNKNOWN, "collectionRequests", "go", "handler", new DumpRequestHandler(), "params", new MapSolrParams(singletonMap("key", "Val2"))), STATUS_OK);
    checkRules(makeMap("resource", "/x", "userPrincipal", "joe", "requestType", RequestType.UNKNOWN, "collectionRequests", "go", "handler", new DumpRequestHandler(), "params", new MapSolrParams(singletonMap("key", "VAL2"))), FORBIDDEN);
    checkRules(makeMap("resource", "/update", "userPrincipal", "solr", "requestType", RequestType.UNKNOWN, "collectionRequests", "go", "handler", new UpdateRequestHandler(), "params", new MapSolrParams(singletonMap("key", "VAL2"))), FORBIDDEN, (Map<String, Object>) Utils.fromJSONString("{user-role:{" + "      admin:[admin_role]," + "      update:[update_role]," + "      solr:[read_role]}," + "    permissions:[" + "      {name:update, role:[admin_role,update_role]}," + "      {name:read, role:[admin_role,update_role,read_role]}" + "]}"));
}

Example 3 with ReplicationHandler

use of org.apache.solr.handler.ReplicationHandler in project lucene-solr by apache.

the class RecoveryStrategy method replicate.

private final void replicate(String nodeName, SolrCore core, ZkNodeProps leaderprops) throws SolrServerException, IOException {
    final String leaderUrl = getReplicateLeaderUrl(leaderprops);
    LOG.info("Attempting to replicate from [{}].", leaderUrl);
    // send commit
    commitOnLeader(leaderUrl);
    // use rep handler directly, so we can do this sync rather than async
    SolrRequestHandler handler = core.getRequestHandler(ReplicationHandler.PATH);
    ReplicationHandler replicationHandler = (ReplicationHandler) handler;
    if (replicationHandler == null) {
        throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Skipping recovery, no " + ReplicationHandler.PATH + " handler found");
    }
    ModifiableSolrParams solrParams = new ModifiableSolrParams();
    solrParams.set(ReplicationHandler.MASTER_URL, leaderUrl);
    // we check closed on return
    if (isClosed())
        return;
    boolean success = replicationHandler.doFetch(solrParams, false).getSuccessful();
    if (!success) {
        throw new SolrException(ErrorCode.SERVER_ERROR, "Replication for recovery failed.");
    }
    // solrcloud_debug
    if (LOG.isDebugEnabled()) {
        try {
            RefCounted<SolrIndexSearcher> searchHolder = core.getNewestSearcher(false);
            SolrIndexSearcher searcher = searchHolder.get();
            Directory dir = core.getDirectoryFactory().get(core.getIndexDir(), DirContext.META_DATA, null);
            try {
                LOG.debug(core.getCoreContainer().getZkController().getNodeName() + " replicated " + searcher.search(new MatchAllDocsQuery(), 1).totalHits + " from " + leaderUrl + " gen:" + (core.getDeletionPolicy().getLatestCommit() != null ? "null" : core.getDeletionPolicy().getLatestCommit().getGeneration()) + " data:" + core.getDataDir() + " index:" + core.getIndexDir() + " newIndex:" + core.getNewIndexDir() + " files:" + Arrays.asList(dir.listAll()));
            } finally {
                core.getDirectoryFactory().release(dir);
                searchHolder.decref();
            }
        } catch (Exception e) {
            LOG.debug("Error in solrcloud_debug block", e);
        }
    }
}