Examples with DocRouter org.apache.solr.common.cloud.DocRouter used on opensource projects

Example 1 with DocRouter

use of org.apache.solr.common.cloud.DocRouter in project lucene-solr by apache.

the class TestHashPartitioner method doIndex.

public void doIndex(DocCollection coll, String id, String expectedShard) {
    DocRouter router = coll.getRouter();
    Slice target = router.getTargetSlice(id, null, null, null, coll);
    assertEquals(expectedShard, target.getName());
}

Example 2 with DocRouter

use of org.apache.solr.common.cloud.DocRouter in project lucene-solr by apache.

the class TestHashPartitioner method doQuery.

public void doQuery(DocCollection coll, String id, String expectedShards) {
    DocRouter router = coll.getRouter();
    Collection<Slice> slices = router.getSearchSlices(id, null, coll);
    List<String> expectedShardStr = StrUtils.splitSmart(expectedShards, ",", true);
    HashSet<String> expectedSet = new HashSet<>(expectedShardStr);
    HashSet<String> obtainedSet = new HashSet<>();
    for (Slice slice : slices) {
        obtainedSet.add(slice.getName());
    }
    // make sure no repeated slices
    assertEquals(slices.size(), obtainedSet.size());
    assertEquals(expectedSet, obtainedSet);
}

Example 3 with DocRouter

use of org.apache.solr.common.cloud.DocRouter in project lucene-solr by apache.

the class TestHashPartitioner method testNonConformingCompositeIds.

/** Make sure CompositeIdRouter doesn't throw exceptions for non-conforming IDs */
public void testNonConformingCompositeIds() throws Exception {
    DocRouter router = DocRouter.getDocRouter(CompositeIdRouter.NAME);
    DocCollection coll = createCollection(4, router);
    String[] ids = { "A!B!C!D", "!!!!!!", "A!!!!B", "A!!B!!C", "A/59!B", "A/8/!B/19/", "A!B/-5", "!/130!", "!!A/1000", "A//8!B///10!C////" };
    for (int i = 0; i < ids.length; ++i) {
        try {
            Slice targetSlice = coll.getRouter().getTargetSlice(ids[i], null, null, null, coll);
            assertNotNull(targetSlice);
        } catch (Exception e) {
            throw new Exception("Exception routing id '" + ids[i] + "'", e);
        }
    }
}

Example 4 with DocRouter

use of org.apache.solr.common.cloud.DocRouter in project lucene-solr by apache.

the class ShardSplitTest method splitByRouteFieldTest.

public void splitByRouteFieldTest() throws Exception {
    log.info("Starting testSplitWithRouteField");
    String collectionName = "routeFieldColl";
    int numShards = 4;
    int replicationFactor = 2;
    int maxShardsPerNode = (((numShards * replicationFactor) / getCommonCloudSolrClient().getZkStateReader().getClusterState().getLiveNodes().size())) + 1;
    HashMap<String, List<Integer>> collectionInfos = new HashMap<>();
    String shard_fld = "shard_s";
    try (CloudSolrClient client = createCloudClient(null)) {
        Map<String, Object> props = Utils.makeMap(REPLICATION_FACTOR, replicationFactor, MAX_SHARDS_PER_NODE, maxShardsPerNode, NUM_SLICES, numShards, "router.field", shard_fld);
        createCollection(collectionInfos, collectionName, props, client);
    }
    List<Integer> list = collectionInfos.get(collectionName);
    checkForCollection(collectionName, list, null);
    waitForRecoveriesToFinish(false);
    String url = getUrlFromZk(getCommonCloudSolrClient().getZkStateReader().getClusterState(), collectionName);
    try (HttpSolrClient collectionClient = getHttpSolrClient(url)) {
        ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
        final DocRouter router = clusterState.getCollection(collectionName).getRouter();
        Slice shard1 = clusterState.getSlice(collectionName, SHARD1);
        DocRouter.Range shard1Range = shard1.getRange() != null ? shard1.getRange() : router.fullRange();
        final List<DocRouter.Range> ranges = router.partitionRange(2, shard1Range);
        final int[] docCounts = new int[ranges.size()];
        for (int i = 100; i <= 200; i++) {
            // See comment in ShardRoutingTest for hash distribution
            String shardKey = "" + (char) ('a' + (i % 26));
            collectionClient.add(getDoc(id, i, "n_ti", i, shard_fld, shardKey));
            int idx = getHashRangeIdx(router, ranges, shardKey);
            if (idx != -1) {
                docCounts[idx]++;
            }
        }
        for (int i = 0; i < docCounts.length; i++) {
            int docCount = docCounts[i];
            log.info("Shard {} docCount = {}", "shard1_" + i, docCount);
        }
        collectionClient.commit();
        for (int i = 0; i < 3; i++) {
            try {
                splitShard(collectionName, SHARD1, null, null);
                break;
            } catch (HttpSolrClient.RemoteSolrException e) {
                if (e.code() != 500) {
                    throw e;
                }
                log.error("SPLITSHARD failed. " + (i < 2 ? " Retring split" : ""), e);
                if (i == 2) {
                    fail("SPLITSHARD was not successful even after three tries");
                }
            }
        }
        waitForRecoveriesToFinish(collectionName, false);
        assertEquals(docCounts[0], collectionClient.query(new SolrQuery("*:*").setParam("shards", "shard1_0")).getResults().getNumFound());
        assertEquals(docCounts[1], collectionClient.query(new SolrQuery("*:*").setParam("shards", "shard1_1")).getResults().getNumFound());
    }
}

Example 5 with DocRouter

use of org.apache.solr.common.cloud.DocRouter in project lucene-solr by apache.

the class ShardSplitTest method splitByUniqueKeyTest.

private void splitByUniqueKeyTest() throws Exception {
    ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
    final DocRouter router = clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION).getRouter();
    Slice shard1 = clusterState.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1);
    DocRouter.Range shard1Range = shard1.getRange() != null ? shard1.getRange() : router.fullRange();
    List<DocRouter.Range> subRanges = new ArrayList<>();
    if (usually()) {
        List<DocRouter.Range> ranges = router.partitionRange(4, shard1Range);
        // 75% of range goes to shard1_0 and the rest to shard1_1
        subRanges.add(new DocRouter.Range(ranges.get(0).min, ranges.get(2).max));
        subRanges.add(ranges.get(3));
    } else {
        subRanges = router.partitionRange(2, shard1Range);
    }
    final List<DocRouter.Range> ranges = subRanges;
    final int[] docCounts = new int[ranges.size()];
    int numReplicas = shard1.getReplicas().size();
    del("*:*");
    for (int id = 0; id <= 100; id++) {
        // See comment in ShardRoutingTest for hash distribution
        String shardKey = "" + (char) ('a' + (id % 26));
        indexAndUpdateCount(router, ranges, docCounts, shardKey + "!" + String.valueOf(id), id);
    }
    commit();
    Thread indexThread = new Thread() {

        @Override
        public void run() {
            Random random = random();
            int max = atLeast(random, 401);
            int sleep = atLeast(random, 25);
            log.info("SHARDSPLITTEST: Going to add " + max + " number of docs at 1 doc per " + sleep + "ms");
            Set<String> deleted = new HashSet<>();
            for (int id = 101; id < max; id++) {
                try {
                    indexAndUpdateCount(router, ranges, docCounts, String.valueOf(id), id);
                    Thread.sleep(sleep);
                    if (usually(random)) {
                        String delId = String.valueOf(random.nextInt(id - 101 + 1) + 101);
                        if (deleted.contains(delId))
                            continue;
                        try {
                            deleteAndUpdateCount(router, ranges, docCounts, delId);
                            deleted.add(delId);
                        } catch (Exception e) {
                            log.error("Exception while deleting docs", e);
                        }
                    }
                } catch (Exception e) {
                    log.error("Exception while adding doc id = " + id, e);
                    // do not select this id for deletion ever
                    deleted.add(String.valueOf(id));
                }
            }
        }
    };
    indexThread.start();
    try {
        for (int i = 0; i < 3; i++) {
            try {
                splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, subRanges, null);
                log.info("Layout after split: \n");
                printLayout();
                break;
            } catch (HttpSolrClient.RemoteSolrException e) {
                if (e.code() != 500) {
                    throw e;
                }
                log.error("SPLITSHARD failed. " + (i < 2 ? " Retring split" : ""), e);
                if (i == 2) {
                    fail("SPLITSHARD was not successful even after three tries");
                }
            }
        }
    } finally {
        try {
            indexThread.join();
        } catch (InterruptedException e) {
            log.error("Indexing thread interrupted", e);
        }
    }
    waitForRecoveriesToFinish(true);
    checkDocCountsAndShardStates(docCounts, numReplicas);
}