Examples with LindenIndexRequest com.xiaomi.linden.thrift.common.LindenIndexRequest used on opensource projects

Example 1 with LindenIndexRequest

use of com.xiaomi.linden.thrift.common.LindenIndexRequest in project linden by XiaoMi.

the class TestLindenSortingMergePolicy method handleRequest.

public void handleRequest(String content) throws IOException {
    LindenIndexRequest indexRequest = LindenIndexRequestParser.parse(lindenConfig.getSchema(), content);
    lindenCore.index(indexRequest);
}

Example 2 with LindenIndexRequest

use of com.xiaomi.linden.thrift.common.LindenIndexRequest in project linden by XiaoMi.

the class CoreLindenServiceImpl method index.

@Override
public Future<Response> index(final String content) {
    final Stopwatch sw = Stopwatch.createStarted();
    return instanceExecutorPool.apply(new Function0<Response>() {

        @Override
        public Response apply() {
            Response response = null;
            try {
                long eps = sw.elapsed(TimeUnit.MILLISECONDS);
                if (eps > 10) {
                    LOGGER.warn("Warning: instanceExecutorPool took " + eps + "ms to start index.");
                    if (eps > instanceFuturePoolWaitTimeout) {
                        response = ResponseUtils.buildFailedResponse("Waiting time is too long, " + eps + "ms in instance future pool");
                        return response;
                    }
                }
                LindenIndexRequest indexRequest = LindenIndexRequestParser.parse(config.getSchema(), content);
                response = lindenCore.index(indexRequest);
            } catch (Exception e) {
                String errorStackInfo = Throwables.getStackTraceAsString(e);
                response = ResponseUtils.buildFailedResponse(errorStackInfo);
            } finally {
                if (response.isSuccess()) {
                    LOGGER.info("Instance index request succeeded, content: {}, cost: {} ms.", content, sw.elapsed(TimeUnit.MILLISECONDS));
                } else {
                    LOGGER.error("Instance index request failed, content: {}, cost: {} ms.", content, sw.elapsed(TimeUnit.MILLISECONDS));
                }
                return response;
            }
        }
    });
}

Example 3 with LindenIndexRequest

use of com.xiaomi.linden.thrift.common.LindenIndexRequest in project linden by XiaoMi.

the class LindenMapper method map.

@Override
public void map(Object key, Object value, Context context) throws IOException, InterruptedException {
    LindenIndexRequest indexRequest;
    try {
        indexRequest = LindenIndexRequestParser.parse(lindenConfig.getSchema(), value.toString());
    } catch (Exception e) {
        ExceptionUtils.printRootCauseStackTrace(e);
        throw new IllegalStateException("LindenIndexRequestParser parsing error", e);
    }
    if (indexRequest.getType() != IndexRequestType.INDEX) {
        throw new IllegalStateException("Index request type error");
    }
    Document doc = LindenDocParser.parse(indexRequest.getDoc(), lindenConfig);
    // now we have uid and lucene Doc;
    IntermediateForm form = new IntermediateForm();
    form.process(new Term(lindenConfig.getSchema().getId(), indexRequest.getDoc().getId()), doc, lindenConfig.getIndexAnalyzerInstance(), facetsConfig);
    form.closeWriter();
    int chosenShard = DefaultShardingStrategy.calculateShard(shards.length, indexRequest);
    if (chosenShard >= 0) {
        // insert into one shard
        context.write(shards[chosenShard], form);
    } else {
        logger.error("calculateShard failed for " + value.toString());
        return;
    }
}

Example 4 with LindenIndexRequest

use of com.xiaomi.linden.thrift.common.LindenIndexRequest in project linden by XiaoMi.

the class CoreLindenCluster method index.

@Override
public Response index(String content) throws IOException {
    List<Future<BoxedUnit>> futures = new ArrayList<>();
    List<String> hosts = new ArrayList<>();
    final StringBuilder errorInfo = new StringBuilder();
    LindenIndexRequest indexRequest = LindenIndexRequestParser.parse(lindenConfig.getSchema(), content);
    for (final Map.Entry<Integer, ShardClient> entry : clients.entrySet()) {
        ShardClient shardClient = entry.getValue();
        if (shardClient.isAvailable()) {
            if (shardingStrategy.accept(indexRequest.getId(), indexRequest.getRouteParam(), shardClient.getShardId())) {
                final List<Map.Entry<String, Future<Response>>> hostFuturePairs = shardClient.index(content);
                for (final Map.Entry<String, Future<Response>> hostFuturePair : hostFuturePairs) {
                    hosts.add(hostFuturePair.getKey());
                    futures.add(hostFuturePair.getValue().transformedBy(new FutureTransformer<Response, BoxedUnit>() {

                        @Override
                        public BoxedUnit map(Response response) {
                            if (!response.isSuccess()) {
                                LOGGER.error("Shard [{}] host [{}] failed to get index response : {}", entry.getKey(), hostFuturePair.getKey(), response.getError());
                                synchronized (errorInfo) {
                                    errorInfo.append("Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + response.getError() + ";");
                                }
                            }
                            return BoxedUnit.UNIT;
                        }

                        @Override
                        public BoxedUnit handle(Throwable t) {
                            LOGGER.error("Shard [{}] host [{}] failed to get index response : {}", entry.getKey(), hostFuturePair.getKey(), Throwables.getStackTraceAsString(t));
                            synchronized (errorInfo) {
                                errorInfo.append("Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + Throwables.getStackTraceAsString(t) + ";");
                            }
                            return BoxedUnit.UNIT;
                        }
                    }));
                }
            }
        }
    }
    try {
        Future<List<BoxedUnit>> collected = Future.collect(futures);
        if (clusterFutureAwaitTimeout == 0) {
            Await.result(collected);
        } else {
            Await.result(collected, Duration.apply(clusterFutureAwaitTimeout, TimeUnit.MILLISECONDS));
        }
        if (errorInfo.length() > 0) {
            return ResponseUtils.buildFailedResponse("Index failed: " + errorInfo.toString());
        }
        return ResponseUtils.SUCCESS;
    } catch (Exception e) {
        LOGGER.error("Handle request failed, content : {} - {}", content, Throwables.getStackTraceAsString(e));
        LOGGER.error(getHostFutureInfo(hosts, futures));
        return ResponseUtils.buildFailedResponse(e);
    }
}

Example 5 with LindenIndexRequest

use of com.xiaomi.linden.thrift.common.LindenIndexRequest in project linden by XiaoMi.

the class TestHotSwapLindenCore method handleRequest.

public void handleRequest(String content) throws IOException {
    LindenIndexRequest indexRequest = LindenIndexRequestParser.parse(lindenConfig.getSchema(), content);
    lindenCore.index(indexRequest);
}