Examples with ShardRouteParam com.xiaomi.linden.thrift.common.ShardRouteParam used on opensource projects

Example 1 with ShardRouteParam

use of com.xiaomi.linden.thrift.common.ShardRouteParam in project linden by XiaoMi.

the class TestBQL method testRoute.

@Test
public void testRoute() {
    String bql = "SELECT * FROM linden where title = 'sed' route by 0, 1, 2, replica_key '12345'";
    LindenSearchRequest lindenRequest = compiler.compile(bql).getSearchRequest();
    SearchRouteParam routeParam = new SearchRouteParam();
    routeParam.addToShardParams(new ShardRouteParam(0));
    routeParam.addToShardParams(new ShardRouteParam(1));
    routeParam.addToShardParams(new ShardRouteParam(2));
    routeParam.setReplicaRouteKey("12345");
    Assert.assertEquals(routeParam, lindenRequest.getRouteParam());
    bql = "SELECT * FROM linden where title = 'sed' route by 0 in top 500, 1, 2, $a, $shard in top 500, 3 in top $max";
    lindenRequest = compiler.compile(bql).getSearchRequest();
    routeParam = new SearchRouteParam();
    routeParam.addToShardParams(new ShardRouteParam(0).setEarlyParam(new EarlyParam(500)));
    routeParam.addToShardParams(new ShardRouteParam(1));
    routeParam.addToShardParams(new ShardRouteParam(2));
    routeParam.addToShardParams(new ShardRouteParam(3));
    Assert.assertEquals(routeParam, lindenRequest.getRouteParam());
    bql = "SELECT * FROM linden where title = 'sed' route by (0,1) in top 10000, ($shard) in top 1000, 2";
    lindenRequest = compiler.compile(bql).getSearchRequest();
    routeParam = new SearchRouteParam();
    routeParam.addToShardParams(new ShardRouteParam(0).setEarlyParam(new EarlyParam().setMaxNum(10000)));
    routeParam.addToShardParams(new ShardRouteParam(1).setEarlyParam(new EarlyParam().setMaxNum(10000)));
    routeParam.addToShardParams(new ShardRouteParam(2));
    Assert.assertEquals(routeParam, lindenRequest.getRouteParam());
}

Example 2 with ShardRouteParam

use of com.xiaomi.linden.thrift.common.ShardRouteParam in project linden by XiaoMi.

the class BQLCompilerAnalyzer method exitRoute_multi_shard_values.

@Override
public void exitRoute_multi_shard_values(BQLParser.Route_multi_shard_valuesContext ctx) {
    List<ShardRouteParam> params = new ArrayList<>();
    for (BQLParser.Numeric_valueContext shardCtx : ctx.numeric_value()) {
        if (shardCtx.PLACEHOLDER() != null) {
            continue;
        }
        ShardRouteParam shardRouteParam = new ShardRouteParam();
        int shardId = Integer.valueOf(shardCtx.getText());
        shardRouteParam.setShardId(shardId);
        EarlyParam earlyParam = (EarlyParam) valProperty.get(ctx.in_top_clause());
        if (earlyParam != null) {
            shardRouteParam.setEarlyParam(earlyParam);
        }
        params.add(shardRouteParam);
    }
    if (!params.isEmpty()) {
        valProperty.put(ctx, params);
    }
}

Example 3 with ShardRouteParam

use of com.xiaomi.linden.thrift.common.ShardRouteParam in project linden by XiaoMi.

the class CoreLindenCluster method delete.

@Override
public Response delete(LindenDeleteRequest request) throws IOException {
    List<Future<BoxedUnit>> futures = new ArrayList<>();
    List<String> hosts = new ArrayList<>();
    final StringBuilder errorInfo = new StringBuilder();
    Set<Integer> routeIds = null;
    if (request.isSetRouteParam()) {
        routeIds = new HashSet<>();
        for (final ShardRouteParam routeParam : request.getRouteParam().getShardParams()) {
            routeIds.add(routeParam.getShardId());
        }
    }
    for (final Map.Entry<Integer, ShardClient> entry : clients.entrySet()) {
        if (routeIds != null && !routeIds.contains(entry.getKey())) {
            continue;
        }
        if (entry.getValue().isAvailable()) {
            List<Map.Entry<String, Future<Response>>> hostFuturePairs = entry.getValue().delete(request);
            for (final Map.Entry<String, Future<Response>> hostFuturePair : hostFuturePairs) {
                hosts.add(hostFuturePair.getKey());
                futures.add(hostFuturePair.getValue().transformedBy(new FutureTransformer<Response, BoxedUnit>() {

                    @Override
                    public BoxedUnit map(Response response) {
                        if (!response.isSuccess()) {
                            synchronized (errorInfo) {
                                LOGGER.error("Shard [{}] host [{}] failed to get delete response : {}", entry.getKey(), hostFuturePair.getKey(), response.getError());
                                errorInfo.append("Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + response.getError() + ";");
                            }
                        }
                        return BoxedUnit.UNIT;
                    }

                    @Override
                    public BoxedUnit handle(Throwable t) {
                        LOGGER.error("Shard [{}] host [{}] failed to get delete response : {}", entry.getKey(), hostFuturePair.getKey(), Throwables.getStackTraceAsString(t));
                        errorInfo.append("Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + Throwables.getStackTraceAsString(t) + ";");
                        return BoxedUnit.UNIT;
                    }
                }));
            }
        }
    }
    Future<List<BoxedUnit>> collected = Future.collect(futures);
    try {
        if (clusterFutureAwaitTimeout == 0) {
            Await.result(collected);
        } else {
            Await.result(collected, Duration.apply(clusterFutureAwaitTimeout, TimeUnit.MILLISECONDS));
        }
        if (errorInfo.length() > 0) {
            return ResponseUtils.buildFailedResponse("Delete failed: " + errorInfo.toString());
        }
        return ResponseUtils.SUCCESS;
    } catch (Exception e) {
        LOGGER.error("Failed to get all delete responses, exception: {}", Throwables.getStackTraceAsString(e));
        LOGGER.error(getHostFutureInfo(hosts, futures));
        return ResponseUtils.buildFailedResponse(e);
    }
}

Example 4 with ShardRouteParam

use of com.xiaomi.linden.thrift.common.ShardRouteParam in project linden by XiaoMi.

the class CoreLindenCluster method coreSearch.

public LindenResult coreSearch(final LindenSearchRequest request) throws IOException {
    List<Future<BoxedUnit>> futures = new ArrayList<>();
    List<String> hosts = new ArrayList<>();
    final List<LindenResult> resultList = new ArrayList<>();
    if (request.isSetRouteParam() && request.getRouteParam().isSetShardParams()) {
        for (final ShardRouteParam routeParam : request.getRouteParam().getShardParams()) {
            ShardClient client = clients.get(routeParam.getShardId());
            if (client != null && client.isAvailable()) {
                LindenSearchRequest subRequest = request;
                if (routeParam.isSetEarlyParam()) {
                    subRequest = new LindenSearchRequest(request);
                    subRequest.setEarlyParam(routeParam.getEarlyParam());
                }
                final Map.Entry<String, Future<LindenResult>> hostFuturePair = client.search(subRequest);
                hosts.add(hostFuturePair.getKey());
                futures.add(hostFuturePair.getValue().transformedBy(new FutureTransformer<LindenResult, BoxedUnit>() {

                    @Override
                    public BoxedUnit map(LindenResult lindenResult) {
                        synchronized (resultList) {
                            resultList.add(lindenResult);
                            if (!lindenResult.isSuccess()) {
                                LOGGER.error("Shard [{}] host [{}] failed to get search result : {}", routeParam.getShardId(), hostFuturePair.getKey(), lindenResult.getError());
                            }
                        }
                        return BoxedUnit.UNIT;
                    }

                    @Override
                    public BoxedUnit handle(Throwable t) {
                        LOGGER.error("Shard [{}] host [{}] failed to get search result : {}", routeParam.getShardId(), hostFuturePair.getKey(), Throwables.getStackTraceAsString(t));
                        return BoxedUnit.UNIT;
                    }
                }));
            } else {
                LOGGER.warn("Route to Shard [{}] failed.", routeParam.getShardId());
            }
        }
    } else {
        LindenSearchRequest subRequest = request;
        for (final Map.Entry<Integer, ShardClient> entry : clients.entrySet()) {
            if (entry.getValue().isAvailable()) {
                final Map.Entry<String, Future<LindenResult>> hostFuturePair = entry.getValue().search(subRequest);
                hosts.add(hostFuturePair.getKey());
                futures.add(hostFuturePair.getValue().transformedBy(new FutureTransformer<LindenResult, BoxedUnit>() {

                    @Override
                    public BoxedUnit map(LindenResult lindenResult) {
                        synchronized (resultList) {
                            resultList.add(lindenResult);
                            if (!lindenResult.isSuccess()) {
                                LOGGER.error("Shard [{}] host [{}] failed to get search result : {}", entry.getKey(), hostFuturePair.getKey(), lindenResult.getError());
                            }
                            return BoxedUnit.UNIT;
                        }
                    }

                    @Override
                    public BoxedUnit handle(Throwable t) {
                        LOGGER.error("Shard [{}] host [{}] failed to get search result : {}", entry.getKey(), hostFuturePair.getKey(), Throwables.getStackTraceAsString(t));
                        return BoxedUnit.UNIT;
                    }
                }));
            }
        }
    }
    Future<List<BoxedUnit>> collected = Future.collect(futures);
    try {
        if (clusterFutureAwaitTimeout == 0) {
            Await.result(collected);
        } else {
            Await.result(collected, Duration.apply(clusterFutureAwaitTimeout, TimeUnit.MILLISECONDS));
        }
    } catch (Exception e) {
        LOGGER.error("Failed to get all results, exception: {}", Throwables.getStackTraceAsString(e));
        LOGGER.error(getHostFutureInfo(hosts, futures));
        if (resultList.size() == 0) {
            return new LindenResult().setSuccess(false).setError("Failed to get any shard result, " + Throwables.getStackTraceAsString(e));
        }
    }
    return ResultMerger.merge(request, resultList);
}

Example 5 with ShardRouteParam

use of com.xiaomi.linden.thrift.common.ShardRouteParam in project linden by XiaoMi.

the class BQLCompilerAnalyzer method exitRoute_single_shard_value.

@Override
public void exitRoute_single_shard_value(BQLParser.Route_single_shard_valueContext ctx) {
    // ignore
    if (ctx.numeric_value().PLACEHOLDER() != null) {
        return;
    }
    ShardRouteParam shardRouteParam = new ShardRouteParam();
    int shardId = Integer.valueOf(ctx.numeric_value().getText());
    shardRouteParam.setShardId(shardId);
    EarlyParam earlyParam = (EarlyParam) valProperty.get(ctx.in_top_clause());
    if (earlyParam != null) {
        shardRouteParam.setEarlyParam(earlyParam);
    }
    valProperty.put(ctx, shardRouteParam);
}