Examples with Builder org.elasticsearch.common.settings.Settings.Builder used on opensource projects

Example 26 with Builder

use of org.elasticsearch.common.settings.Settings.Builder in project elasticsearch by elastic.

the class QueryRescorerIT method testRescorerMadeScoresWorse.

// Tests a rescorer that penalizes the scores:
public void testRescorerMadeScoresWorse() throws Exception {
    Builder builder = Settings.builder();
    builder.put("index.analysis.analyzer.synonym.tokenizer", "whitespace");
    builder.putArray("index.analysis.analyzer.synonym.filter", "synonym", "lowercase");
    builder.put("index.analysis.filter.synonym.type", "synonym");
    builder.putArray("index.analysis.filter.synonym.synonyms", "ave => ave, avenue", "street => str, street");
    XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("field1").field("type", "text").field("analyzer", "whitespace").field("search_analyzer", "synonym").endObject().endObject().endObject().endObject();
    assertAcked(client().admin().indices().prepareCreate("test").addMapping("type1", mapping).setSettings(builder.put("index.number_of_shards", 1)));
    client().prepareIndex("test", "type1", "3").setSource("field1", "massachusetts").execute().actionGet();
    client().prepareIndex("test", "type1", "6").setSource("field1", "massachusetts avenue lexington massachusetts").execute().actionGet();
    client().admin().indices().prepareRefresh("test").execute().actionGet();
    client().prepareIndex("test", "type1", "1").setSource("field1", "lexington massachusetts avenue").execute().actionGet();
    client().prepareIndex("test", "type1", "2").setSource("field1", "lexington avenue boston massachusetts road").execute().actionGet();
    client().admin().indices().prepareRefresh("test").execute().actionGet();
    SearchResponse searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchQuery("field1", "massachusetts").operator(Operator.OR)).setFrom(0).setSize(5).execute().actionGet();
    assertThat(searchResponse.getHits().getHits().length, equalTo(4));
    assertHitCount(searchResponse, 4);
    assertThat(searchResponse.getHits().getMaxScore(), equalTo(searchResponse.getHits().getHits()[0].getScore()));
    assertFirstHit(searchResponse, hasId("3"));
    assertSecondHit(searchResponse, hasId("6"));
    assertThirdHit(searchResponse, hasId("1"));
    assertFourthHit(searchResponse, hasId("2"));
    // Now, penalizing rescore (nothing matches the rescore query):
    searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchQuery("field1", "massachusetts").operator(Operator.OR)).setFrom(0).setSize(5).setRescorer(queryRescorer(QueryBuilders.matchPhraseQuery("field1", "lexington avenue massachusetts").slop(3)).setQueryWeight(1.0f).setRescoreQueryWeight(-1f), 3).execute().actionGet();
    // 6 and 1 got worse, and then the hit (2) outside the rescore window were sorted ahead:
    assertThat(searchResponse.getHits().getMaxScore(), equalTo(searchResponse.getHits().getHits()[0].getScore()));
    assertFirstHit(searchResponse, hasId("3"));
    assertSecondHit(searchResponse, hasId("2"));
    assertThirdHit(searchResponse, hasId("6"));
    assertFourthHit(searchResponse, hasId("1"));
}

Example 27 with Builder

use of org.elasticsearch.common.settings.Settings.Builder in project elasticsearch by elastic.

the class SearchWithRandomExceptionsIT method testRandomExceptions.

public void testRandomExceptions() throws IOException, InterruptedException, ExecutionException {
    String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties").startObject("test").field("type", "keyword").endObject().endObject().endObject().endObject().string();
    final double lowLevelRate;
    final double topLevelRate;
    if (frequently()) {
        if (randomBoolean()) {
            if (randomBoolean()) {
                lowLevelRate = 1.0 / between(2, 10);
                topLevelRate = 0.0d;
            } else {
                topLevelRate = 1.0 / between(2, 10);
                lowLevelRate = 0.0d;
            }
        } else {
            lowLevelRate = 1.0 / between(2, 10);
            topLevelRate = 1.0 / between(2, 10);
        }
    } else {
        // rarely no exception
        topLevelRate = 0d;
        lowLevelRate = 0d;
    }
    Builder settings = Settings.builder().put(indexSettings()).put(EXCEPTION_TOP_LEVEL_RATIO_KEY, topLevelRate).put(EXCEPTION_LOW_LEVEL_RATIO_KEY, lowLevelRate).put(MockEngineSupport.WRAP_READER_RATIO.getKey(), 1.0d);
    logger.info("creating index: [test] using settings: [{}]", settings.build().getAsMap());
    assertAcked(prepareCreate("test").setSettings(settings).addMapping("type", mapping, XContentType.JSON));
    ensureSearchable();
    final int numDocs = between(10, 100);
    int numCreated = 0;
    boolean[] added = new boolean[numDocs];
    for (int i = 0; i < numDocs; i++) {
        try {
            IndexResponse indexResponse = client().prepareIndex("test", "type", "" + i).setTimeout(TimeValue.timeValueSeconds(1)).setSource("test", English.intToEnglish(i)).get();
            if (indexResponse.getResult() == DocWriteResponse.Result.CREATED) {
                numCreated++;
                added[i] = true;
            }
        } catch (ElasticsearchException ex) {
        }
    }
    logger.info("Start Refresh");
    // don't assert on failures here
    RefreshResponse refreshResponse = client().admin().indices().prepareRefresh("test").execute().get();
    final boolean refreshFailed = refreshResponse.getShardFailures().length != 0 || refreshResponse.getFailedShards() != 0;
    logger.info("Refresh failed [{}] numShardsFailed: [{}], shardFailuresLength: [{}], successfulShards: [{}], totalShards: [{}] ", refreshFailed, refreshResponse.getFailedShards(), refreshResponse.getShardFailures().length, refreshResponse.getSuccessfulShards(), refreshResponse.getTotalShards());
    NumShards test = getNumShards("test");
    final int numSearches = scaledRandomIntBetween(100, 200);
    // we don't check anything here really just making sure we don't leave any open files or a broken index behind.
    for (int i = 0; i < numSearches; i++) {
        try {
            int docToQuery = between(0, numDocs - 1);
            int expectedResults = added[docToQuery] ? 1 : 0;
            logger.info("Searching for [test:{}]", English.intToEnglish(docToQuery));
            SearchResponse searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchQuery("test", English.intToEnglish(docToQuery))).setSize(expectedResults).get();
            logger.info("Successful shards: [{}]  numShards: [{}]", searchResponse.getSuccessfulShards(), test.numPrimaries);
            if (searchResponse.getSuccessfulShards() == test.numPrimaries && !refreshFailed) {
                assertResultsAndLogOnFailure(expectedResults, searchResponse);
            }
            // check match all
            searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchAllQuery()).setSize(numCreated).addSort("_id", SortOrder.ASC).get();
            logger.info("Match all Successful shards: [{}]  numShards: [{}]", searchResponse.getSuccessfulShards(), test.numPrimaries);
            if (searchResponse.getSuccessfulShards() == test.numPrimaries && !refreshFailed) {
                assertResultsAndLogOnFailure(numCreated, searchResponse);
            }
        } catch (SearchPhaseExecutionException ex) {
            logger.info("expected SearchPhaseException: [{}]", ex.getMessage());
        }
    }
}

Example 28 with Builder

use of org.elasticsearch.common.settings.Settings.Builder in project elasticsearch by elastic.

the class InternalTestCluster method reset.

private synchronized void reset(boolean wipeData) throws IOException {
    // clear all rules for mock transport services
    for (NodeAndClient nodeAndClient : nodes.values()) {
        TransportService transportService = nodeAndClient.node.injector().getInstance(TransportService.class);
        if (transportService instanceof MockTransportService) {
            final MockTransportService mockTransportService = (MockTransportService) transportService;
            mockTransportService.clearAllRules();
            mockTransportService.clearTracers();
        }
    }
    randomlyResetClients();
    final int newSize = sharedNodesSeeds.length;
    if (nextNodeId.get() == newSize && nodes.size() == newSize) {
        if (wipeData) {
            wipePendingDataDirectories();
        }
        if (nodes.size() > 0 && autoManageMinMasterNodes) {
            updateMinMasterNodes(getMasterNodesCount());
        }
        logger.debug("Cluster hasn't changed - moving out - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), newSize);
        return;
    }
    logger.debug("Cluster is NOT consistent - restarting shared nodes - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), newSize);
    // trash all nodes with id >= sharedNodesSeeds.length - they are non shared
    final List<NodeAndClient> toClose = new ArrayList<>();
    for (Iterator<NodeAndClient> iterator = nodes.values().iterator(); iterator.hasNext(); ) {
        NodeAndClient nodeAndClient = iterator.next();
        if (nodeAndClient.nodeAndClientId() >= sharedNodesSeeds.length) {
            logger.debug("Close Node [{}] not shared", nodeAndClient.name);
            toClose.add(nodeAndClient);
        }
    }
    stopNodesAndClients(toClose);
    // clean up what the nodes left that is unused
    if (wipeData) {
        wipePendingDataDirectories();
    }
    // start any missing node
    assert newSize == numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes;
    final int numberOfMasterNodes = numSharedDedicatedMasterNodes > 0 ? numSharedDedicatedMasterNodes : numSharedDataNodes;
    final int defaultMinMasterNodes = (numberOfMasterNodes / 2) + 1;
    // we want to start nodes in one go due to min master nodes
    final List<NodeAndClient> toStartAndPublish = new ArrayList<>();
    for (int i = 0; i < numSharedDedicatedMasterNodes; i++) {
        final Settings.Builder settings = Settings.builder();
        settings.put(Node.NODE_MASTER_SETTING.getKey(), true);
        settings.put(Node.NODE_DATA_SETTING.getKey(), false);
        NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true, defaultMinMasterNodes);
        toStartAndPublish.add(nodeAndClient);
    }
    for (int i = numSharedDedicatedMasterNodes; i < numSharedDedicatedMasterNodes + numSharedDataNodes; i++) {
        final Settings.Builder settings = Settings.builder();
        if (numSharedDedicatedMasterNodes > 0) {
            // if we don't have dedicated master nodes, keep things default
            settings.put(Node.NODE_MASTER_SETTING.getKey(), false).build();
            settings.put(Node.NODE_DATA_SETTING.getKey(), true).build();
        }
        NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true, defaultMinMasterNodes);
        toStartAndPublish.add(nodeAndClient);
    }
    for (int i = numSharedDedicatedMasterNodes + numSharedDataNodes; i < numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; i++) {
        final Builder settings = Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false).put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_INGEST_SETTING.getKey(), false);
        NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true, defaultMinMasterNodes);
        toStartAndPublish.add(nodeAndClient);
    }
    startAndPublishNodesAndClients(toStartAndPublish);
    nextNodeId.set(newSize);
    assert size() == newSize;
    if (newSize > 0) {
        validateClusterFormed();
    }
    logger.debug("Cluster is consistent again - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), newSize);
}

Example 29 with Builder

use of org.elasticsearch.common.settings.Settings.Builder in project fess by codelibs.

the class FessEsClient method open.

@PostConstruct
public void open() {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final String transportAddressesValue = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
    if (StringUtil.isNotBlank(transportAddressesValue)) {
        for (final String transportAddressValue : transportAddressesValue.split(",")) {
            final String[] addressPair = transportAddressValue.trim().split(":");
            if (addressPair.length < 3) {
                final String host = addressPair[0];
                int port = 9300;
                if (addressPair.length == 2) {
                    port = Integer.parseInt(addressPair[1]);
                }
                addTransportAddress(host, port);
            } else {
                logger.warn("Invalid address format: " + transportAddressValue);
            }
        }
    }
    if (transportAddressList.isEmpty()) {
        if (runner == null) {
            runner = new ElasticsearchClusterRunner();
            final Configs config = newConfigs().clusterName(fessConfig.getElasticsearchClusterName()).numOfNode(1).useLogger();
            final String esDir = System.getProperty("fess.es.dir");
            if (esDir != null) {
                config.basePath(esDir);
            }
            config.disableESLogger();
            runner.onBuild((number, settingsBuilder) -> {
                final File pluginDir = new File(esDir, "plugins");
                if (pluginDir.isDirectory()) {
                    settingsBuilder.put("path.plugins", pluginDir.getAbsolutePath());
                } else {
                    settingsBuilder.put("path.plugins", new File(System.getProperty("user.dir"), "plugins").getAbsolutePath());
                }
                if (settings != null) {
                    settingsBuilder.put(settings);
                }
            });
            runner.build(config);
        }
        client = runner.client();
        addTransportAddress("localhost", runner.node().settings().getAsInt("transport.tcp.port", 9300));
    } else {
        final Builder settingsBuilder = Settings.builder();
        settingsBuilder.put("cluster.name", fessConfig.getElasticsearchClusterName());
        settingsBuilder.put("client.transport.sniff", fessConfig.isElasticsearchTransportSniff());
        settingsBuilder.put("client.transport.ping_timeout", fessConfig.getElasticsearchTransportPingTimeout());
        settingsBuilder.put("client.transport.nodes_sampler_interval", fessConfig.getElasticsearchTransportNodesSamplerInterval());
        final Settings settings = settingsBuilder.build();
        final TransportClient transportClient = new PreBuiltTransportClient(settings);
        for (final TransportAddress address : transportAddressList) {
            transportClient.addTransportAddress(address);
        }
        client = transportClient;
    }
    if (StringUtil.isBlank(transportAddressesValue)) {
        final StringBuilder buf = new StringBuilder();
        for (final TransportAddress transportAddress : transportAddressList) {
            if (transportAddress instanceof InetSocketTransportAddress) {
                if (buf.length() > 0) {
                    buf.append(',');
                }
                final InetSocketTransportAddress inetTransportAddress = (InetSocketTransportAddress) transportAddress;
                buf.append(inetTransportAddress.address().getHostName());
                buf.append(':');
                buf.append(inetTransportAddress.address().getPort());
            }
        }
        if (buf.length() > 0) {
            System.setProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES, buf.toString());
        }
    }
    waitForYellowStatus();
    indexConfigList.forEach(configName -> {
        final String[] values = configName.split("/");
        if (values.length == 2) {
            final String configIndex = values[0];
            final String configType = values[1];
            final String indexName;
            final boolean isFessIndex = configIndex.equals("fess");
            if (isFessIndex) {
                indexName = fessConfig.getIndexDocumentUpdateIndex();
            } else {
                indexName = configIndex;
            }
            boolean exists = existsIndex(indexName);
            if (!exists) {
                final String createdIndexName;
                if (isFessIndex) {
                    createdIndexName = generateNewIndexName(configIndex);
                } else {
                    createdIndexName = configIndex;
                }
                createIndex(configIndex, configType, createdIndexName);
                createAlias(configIndex, createdIndexName);
            }
            final String updatedIndexName;
            if (isFessIndex) {
                client.admin().cluster().prepareHealth(fessConfig.getIndexDocumentUpdateIndex()).setWaitForYellowStatus().execute().actionGet(fessConfig.getIndexIndicesTimeout());
                final GetIndexResponse response = client.admin().indices().prepareGetIndex().addIndices(fessConfig.getIndexDocumentUpdateIndex()).execute().actionGet(fessConfig.getIndexIndicesTimeout());
                final String[] indices = response.indices();
                if (indices.length == 1) {
                    updatedIndexName = indices[0];
                } else {
                    updatedIndexName = configIndex;
                }
            } else {
                updatedIndexName = configIndex;
            }
            addMapping(configIndex, configType, updatedIndexName);
        } else {
            logger.warn("Invalid index config name: " + configName);
        }
    });
}

Example 30 with Builder

use of org.elasticsearch.common.settings.Settings.Builder in project springboot_op by SnailFastGo.

the class ElasticsearchConfiguration method initESClient.

@Bean
public TransportClient initESClient() throws NumberFormatException, UnknownHostException {
    String ip = env.getProperty("spring.es.ip");
    String port = env.getProperty("spring.es.port");
    String clusterName = env.getProperty("spring.es.cluster_name");
    Builder builder = Settings.builder().put("cluster.name", clusterName).put("client.transport.sniff", true);
    Settings esSettings = builder.build();
    TransportClient client = new PreBuiltTransportClient(esSettings);
    client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(ip), Integer.parseInt(port)));
    logger.info("ES Client 初始化成功, ip : {}, port : {}, cluster_name : {}", ip, port, clusterName);
    return client;
}