Examples with CreateIndexResponse org.opensearch.action.admin.indices.create.CreateIndexResponse used on opensource projects

Example 1 with CreateIndexResponse

use of org.opensearch.action.admin.indices.create.CreateIndexResponse in project fess-crawler by codelibs.

the class AbstractCrawlerService method createMapping.

protected void createMapping(final String mappingName) {
    boolean exists = false;
    try {
        final IndicesExistsResponse response = fesenClient.get(c -> c.admin().indices().prepareExists(index).execute());
        exists = response.isExists();
    } catch (final IndexNotFoundException e) {
    // ignore
    }
    if (!exists) {
        final CreateIndexResponse indexResponse = fesenClient.get(c -> {
            final String source;
            if (numberOfReplicas > 0) {
                source = "{\"settings\":{\"index\":{\"number_of_shards\":" + numberOfShards + ",\"number_of_replicas\":0,\"auto_expand_replicas\":\"0-" + numberOfReplicas + "\"}}}";
            } else {
                source = "{\"settings\":{\"index\":{\"number_of_shards\":" + numberOfShards + ",\"number_of_replicas\":" + numberOfReplicas + "}}}";
            }
            return c.admin().indices().prepareCreate(index).setSource(source, XContentType.JSON).execute();
        });
        if (indexResponse.isAcknowledged()) {
            logger.info("Created {} index.", index);
        } else if (logger.isDebugEnabled()) {
            logger.debug("Failed to create {} index.", index);
        }
    }
    final GetMappingsResponse getMappingsResponse = fesenClient.get(c -> c.admin().indices().prepareGetMappings(index).execute());
    final ImmutableOpenMap<String, MappingMetadata> indexMappings = getMappingsResponse.mappings().get(index);
    if (indexMappings == null || !indexMappings.containsKey("properties")) {
        final AcknowledgedResponse putMappingResponse = fesenClient.get(c -> {
            final String source = FileUtil.readText("mapping/" + mappingName + ".json");
            return c.admin().indices().preparePutMapping(index).setSource(source, XContentType.JSON).execute();
        });
        if (putMappingResponse.isAcknowledged()) {
            logger.info("Created {} mapping.", index);
        } else {
            logger.warn("Failed to create {} mapping.", index);
        }
    } else if (logger.isDebugEnabled()) {
        logger.debug("{} mapping exists.", index);
    }
}

Example 2 with CreateIndexResponse

use of org.opensearch.action.admin.indices.create.CreateIndexResponse in project OpenSearch by opensearch-project.

the class TransportBulkAction method doInternalExecute.

protected void doInternalExecute(Task task, BulkRequest bulkRequest, String executorName, ActionListener<BulkResponse> listener) {
    final long startTime = relativeTime();
    final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size());
    boolean hasIndexRequestsWithPipelines = false;
    final Metadata metadata = clusterService.state().getMetadata();
    final Version minNodeVersion = clusterService.state().getNodes().getMinNodeVersion();
    for (DocWriteRequest<?> actionRequest : bulkRequest.requests) {
        IndexRequest indexRequest = getIndexWriteRequest(actionRequest);
        if (indexRequest != null) {
            // Each index request needs to be evaluated, because this method also modifies the IndexRequest
            boolean indexRequestHasPipeline = IngestService.resolvePipelines(actionRequest, indexRequest, metadata);
            hasIndexRequestsWithPipelines |= indexRequestHasPipeline;
        }
        if (actionRequest instanceof IndexRequest) {
            IndexRequest ir = (IndexRequest) actionRequest;
            ir.checkAutoIdWithOpTypeCreateSupportedByVersion(minNodeVersion);
            if (ir.getAutoGeneratedTimestamp() != IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP) {
                throw new IllegalArgumentException("autoGeneratedTimestamp should not be set externally");
            }
        }
    }
    if (hasIndexRequestsWithPipelines) {
        // this path is never taken.
        try {
            if (Assertions.ENABLED) {
                final boolean arePipelinesResolved = bulkRequest.requests().stream().map(TransportBulkAction::getIndexWriteRequest).filter(Objects::nonNull).allMatch(IndexRequest::isPipelineResolved);
                assert arePipelinesResolved : bulkRequest;
            }
            if (clusterService.localNode().isIngestNode()) {
                processBulkIndexIngestRequest(task, bulkRequest, executorName, listener);
            } else {
                ingestForwarder.forwardIngestRequest(BulkAction.INSTANCE, bulkRequest, listener);
            }
        } catch (Exception e) {
            listener.onFailure(e);
        }
        return;
    }
    final boolean includesSystem = includesSystem(bulkRequest, clusterService.state().metadata().getIndicesLookup(), systemIndices);
    if (includesSystem || needToCheck()) {
        // Attempt to create all the indices that we're going to need during the bulk before we start.
        // Step 1: collect all the indices in the request
        final Map<String, Boolean> indices = bulkRequest.requests.stream().filter(request -> request.opType() != DocWriteRequest.OpType.DELETE || request.versionType() == VersionType.EXTERNAL || request.versionType() == VersionType.EXTERNAL_GTE).collect(Collectors.toMap(DocWriteRequest::index, DocWriteRequest::isRequireAlias, (v1, v2) -> v1 || v2));
        /* Step 2: filter that to indices that don't exist and we can create. At the same time build a map of indices we can't create
             * that we'll use when we try to run the requests. */
        final Map<String, IndexNotFoundException> indicesThatCannotBeCreated = new HashMap<>();
        Set<String> autoCreateIndices = new HashSet<>();
        ClusterState state = clusterService.state();
        for (Map.Entry<String, Boolean> indexAndFlag : indices.entrySet()) {
            boolean shouldAutoCreate;
            final String index = indexAndFlag.getKey();
            try {
                shouldAutoCreate = shouldAutoCreate(index, state);
            } catch (IndexNotFoundException e) {
                shouldAutoCreate = false;
                indicesThatCannotBeCreated.put(index, e);
            }
            // We should only auto create if we are not requiring it to be an alias
            if (shouldAutoCreate && (indexAndFlag.getValue() == false)) {
                autoCreateIndices.add(index);
            }
        }
        // Step 3: create all the indices that are missing, if there are any missing. start the bulk after all the creates come back.
        if (autoCreateIndices.isEmpty()) {
            executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
        } else {
            final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size());
            for (String index : autoCreateIndices) {
                createIndex(index, bulkRequest.timeout(), minNodeVersion, new ActionListener<CreateIndexResponse>() {

                    @Override
                    public void onResponse(CreateIndexResponse result) {
                        if (counter.decrementAndGet() == 0) {
                            threadPool.executor(executorName).execute(new ActionRunnable<BulkResponse>(listener) {

                                @Override
                                protected void doRun() {
                                    executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
                                }
                            });
                        }
                    }

                    @Override
                    public void onFailure(Exception e) {
                        if (!(ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException)) {
                            // fail all requests involving this index, if create didn't work
                            for (int i = 0; i < bulkRequest.requests.size(); i++) {
                                DocWriteRequest<?> request = bulkRequest.requests.get(i);
                                if (request != null && setResponseFailureIfIndexMatches(responses, i, request, index, e)) {
                                    bulkRequest.requests.set(i, null);
                                }
                            }
                        }
                        if (counter.decrementAndGet() == 0) {
                            final ActionListener<BulkResponse> wrappedListener = ActionListener.wrap(listener::onResponse, inner -> {
                                inner.addSuppressed(e);
                                listener.onFailure(inner);
                            });
                            threadPool.executor(executorName).execute(new ActionRunnable<BulkResponse>(wrappedListener) {

                                @Override
                                protected void doRun() {
                                    executeBulk(task, bulkRequest, startTime, wrappedListener, responses, indicesThatCannotBeCreated);
                                }

                                @Override
                                public void onRejection(Exception rejectedException) {
                                    rejectedException.addSuppressed(e);
                                    super.onRejection(rejectedException);
                                }
                            });
                        }
                    }
                });
            }
        }
    } else {
        executeBulk(task, bulkRequest, startTime, listener, responses, emptyMap());
    }
}

Example 3 with CreateIndexResponse

use of org.opensearch.action.admin.indices.create.CreateIndexResponse in project OpenSearch by opensearch-project.

the class RandomExceptionCircuitBreakerIT method testBreakerWithRandomExceptions.

public void testBreakerWithRandomExceptions() throws IOException, InterruptedException, ExecutionException {
    for (NodeStats node : client().admin().cluster().prepareNodesStats().clear().addMetric(BREAKER.metricName()).execute().actionGet().getNodes()) {
        assertThat("Breaker is not set to 0", node.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(), equalTo(0L));
    }
    String mapping = // {}
    Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties").startObject("test-str").field("type", "keyword").field("doc_values", randomBoolean()).endObject().startObject("test-num").field("type", randomFrom(Arrays.asList("float", "long", "double", "short", "integer"))).endObject().endObject().endObject().endObject());
    final double topLevelRate;
    final double lowLevelRate;
    if (frequently()) {
        if (randomBoolean()) {
            if (randomBoolean()) {
                lowLevelRate = 1.0 / between(2, 10);
                topLevelRate = 0.0d;
            } else {
                topLevelRate = 1.0 / between(2, 10);
                lowLevelRate = 0.0d;
            }
        } else {
            lowLevelRate = 1.0 / between(2, 10);
            topLevelRate = 1.0 / between(2, 10);
        }
    } else {
        // rarely no exception
        topLevelRate = 0d;
        lowLevelRate = 0d;
    }
    Settings.Builder settings = Settings.builder().put(indexSettings()).put(EXCEPTION_TOP_LEVEL_RATIO_KEY, topLevelRate).put(EXCEPTION_LOW_LEVEL_RATIO_KEY, lowLevelRate).put(MockEngineSupport.WRAP_READER_RATIO.getKey(), 1.0d);
    logger.info("creating index: [test] using settings: [{}]", settings.build());
    CreateIndexResponse response = client().admin().indices().prepareCreate("test").setSettings(settings).addMapping("type", mapping, XContentType.JSON).execute().actionGet();
    final int numDocs;
    if (response.isShardsAcknowledged() == false) {
        /* some seeds just won't let you create the index at all and we enter a ping-pong mode
             * trying one node after another etc. that is ok but we need to make sure we don't wait
             * forever when indexing documents so we set numDocs = 1 and expect all shards to fail
             * when we search below.*/
        if (response.isAcknowledged()) {
            logger.info("Index creation timed out waiting for primaries to start - only index one doc and expect searches to fail");
        } else {
            logger.info("Index creation failed - only index one doc and expect searches to fail");
        }
        numDocs = 1;
    } else {
        numDocs = between(10, 100);
    }
    for (int i = 0; i < numDocs; i++) {
        try {
            client().prepareIndex("test").setId("" + i).setTimeout(TimeValue.timeValueSeconds(1)).setSource("test-str", randomUnicodeOfLengthBetween(5, 25), "test-num", i).get();
        } catch (OpenSearchException ex) {
        }
    }
    logger.info("Start Refresh");
    // don't assert on failures here
    RefreshResponse refreshResponse = client().admin().indices().prepareRefresh("test").execute().get();
    final boolean refreshFailed = refreshResponse.getShardFailures().length != 0 || refreshResponse.getFailedShards() != 0;
    logger.info("Refresh failed: [{}] numShardsFailed: [{}], shardFailuresLength: [{}], successfulShards: [{}], totalShards: [{}] ", refreshFailed, refreshResponse.getFailedShards(), refreshResponse.getShardFailures().length, refreshResponse.getSuccessfulShards(), refreshResponse.getTotalShards());
    final int numSearches = scaledRandomIntBetween(50, 150);
    NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().clear().addMetric(BREAKER.metricName()).execute().actionGet();
    for (NodeStats stats : resp.getNodes()) {
        assertThat("Breaker is set to 0", stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(), equalTo(0L));
    }
    for (int i = 0; i < numSearches; i++) {
        SearchRequestBuilder searchRequestBuilder = client().prepareSearch().setQuery(QueryBuilders.matchAllQuery());
        if (random().nextBoolean()) {
            searchRequestBuilder.addSort("test-str", SortOrder.ASC);
        }
        searchRequestBuilder.addSort("test-num", SortOrder.ASC);
        boolean success = false;
        try {
            // Sort by the string and numeric fields, to load them into field data
            searchRequestBuilder.get();
            success = true;
        } catch (SearchPhaseExecutionException ex) {
            logger.info("expected SearchPhaseException: [{}]", ex.getMessage());
        }
        if (frequently()) {
            // Now, clear the cache and check that the circuit breaker has been
            // successfully set back to zero. If there is a bug in the circuit
            // breaker adjustment code, it should show up here by the breaker
            // estimate being either positive or negative.
            // make sure all shards are there - there could be shards that are still starting up.
            ensureGreen("test");
            assertAllSuccessful(client().admin().indices().prepareClearCache("test").setFieldDataCache(true).execute().actionGet());
            // Since .cleanUp() is no longer called on cache clear, we need to call it on each node manually
            for (String node : internalCluster().getNodeNames()) {
                final IndicesFieldDataCache fdCache = internalCluster().getInstance(IndicesService.class, node).getIndicesFieldDataCache();
                // Clean up the cache, ensuring that entries' listeners have been called
                fdCache.getCache().refresh();
            }
            NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().clear().addMetric(BREAKER.metricName()).execute().actionGet();
            for (NodeStats stats : nodeStats.getNodes()) {
                assertThat("Breaker reset to 0 last search success: " + success + " mapping: " + mapping, stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(), equalTo(0L));
            }
        }
    }
}

Example 4 with CreateIndexResponse

use of org.opensearch.action.admin.indices.create.CreateIndexResponse in project OpenSearch by opensearch-project.

the class ActiveShardsObserverIT method testCreateIndexNoActiveShardsNoWaiting.

public void testCreateIndexNoActiveShardsNoWaiting() throws Exception {
    Settings.Builder settingsBuilder = Settings.builder().put(indexSettings()).put(INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), randomIntBetween(1, 5)).put(INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0);
    if (internalCluster().getNodeNames().length > 0) {
        String exclude = String.join(",", internalCluster().getNodeNames());
        settingsBuilder.put("index.routing.allocation.exclude._name", exclude);
    }
    Settings settings = settingsBuilder.build();
    CreateIndexResponse response = prepareCreate("test-idx").setSettings(settings).setWaitForActiveShards(ActiveShardCount.NONE).get();
    assertTrue(response.isAcknowledged());
}

Example 5 with CreateIndexResponse

use of org.opensearch.action.admin.indices.create.CreateIndexResponse in project OpenSearch by opensearch-project.

the class ActiveShardsObserverIT method testCreateIndexStopsWaitingWhenIndexDeleted.

public void testCreateIndexStopsWaitingWhenIndexDeleted() throws Exception {
    final String indexName = "test-idx";
    Settings settings = Settings.builder().put(indexSettings()).put(INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), randomIntBetween(1, 5)).put(INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), internalCluster().numDataNodes() - 1).build();
    logger.info("--> start the index creation process");
    ActionFuture<CreateIndexResponse> responseListener = prepareCreate(indexName).setSettings(settings).setWaitForActiveShards(ActiveShardCount.ALL).execute();
    logger.info("--> wait until the cluster state contains the new index");
    assertBusy(() -> assertTrue(client().admin().cluster().prepareState().get().getState().metadata().hasIndex(indexName)));
    logger.info("--> delete the index");
    assertAcked(client().admin().indices().prepareDelete(indexName));
    logger.info("--> ensure the create index request completes");
    assertAcked(responseListener.get());
}