use of org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.CreateIndexRequest in project hazelcast by hazelcast.
the class CommonElasticSinksTest method given_documentNotInIndex_whenWriteToElasticSinkUpdateRequest_then_jobShouldFail.
/**
* Regression test for checking that behavior was not unexpectedly changed.
* It is possible that behavior will be changed in any of future version
* since failing job based on unsuccessful delete/update leads to problems
* when job are restarted.
*/
@Test
public void given_documentNotInIndex_whenWriteToElasticSinkUpdateRequest_then_jobShouldFail() throws Exception {
elasticClient.indices().create(new CreateIndexRequest("my-index"), RequestOptions.DEFAULT);
Sink<TestItem> elasticSink = new ElasticSinkBuilder<>().clientFn(elasticClientSupplier()).mapToRequestFn((TestItem item) -> new UpdateRequest("my-index", item.getId()).doc(item.asMap())).retries(0).build();
Pipeline p = Pipeline.create();
p.readFrom(TestSources.items(new TestItem("notExist", "Frantisek"))).writeTo(elasticSink);
assertThatThrownBy(() -> submitJob(p)).hasRootCauseInstanceOf(JetException.class).hasStackTraceContaining("document missing");
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.CreateIndexRequest in project sonarqube by SonarSource.
the class IndexCreator method createIndex.
private void createIndex(BuiltIndex<?> builtIndex, boolean useMetadata) {
Index index = builtIndex.getMainType().getIndex();
LOGGER.info(String.format("Create index [%s]", index.getName()));
Settings.Builder settings = Settings.builder();
settings.put(builtIndex.getSettings());
if (useMetadata) {
metadataIndex.setHash(index, IndexDefinitionHash.of(builtIndex));
metadataIndex.setInitialized(builtIndex.getMainType(), false);
builtIndex.getRelationTypes().forEach(relationType -> metadataIndex.setInitialized(relationType, false));
}
CreateIndexResponse indexResponse = client.create(new CreateIndexRequest(index.getName()).settings((settings)));
if (!indexResponse.isAcknowledged()) {
throw new IllegalStateException("Failed to create index [" + index.getName() + "]");
}
client.waitForStatus(ClusterHealthStatus.YELLOW);
// create types
LOGGER.info("Create type {}", builtIndex.getMainType().format());
AcknowledgedResponse mappingResponse = client.putMapping(new PutMappingRequest(builtIndex.getMainType().getIndex().getName()).type(builtIndex.getMainType().getType()).source(builtIndex.getAttributes()));
if (!mappingResponse.isAcknowledged()) {
throw new IllegalStateException("Failed to create type " + builtIndex.getMainType().getType());
}
client.waitForStatus(ClusterHealthStatus.YELLOW);
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.CreateIndexRequest in project snow-owl by b2ihealthcare.
the class EsIndexAdmin method create.
@Override
public void create() {
log.info("Preparing '{}' indexes...", name);
// register any type that requires a refresh at the end of the index create/open
Set<DocumentMapping> mappingsToRefresh = Sets.newHashSet();
// create number of indexes based on number of types
for (DocumentMapping mapping : mappings.getMappings()) {
final String index = getTypeIndex(mapping);
final Map<String, Object> typeMapping = ImmutableMap.<String, Object>builder().put("date_detection", false).put("numeric_detection", false).put("dynamic_templates", List.of(stringsAsKeywords())).putAll(toProperties(mapping)).build();
if (exists(mapping)) {
// update mapping if required
final MappingMetadata currentIndexMapping;
try {
final GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(index);
currentIndexMapping = client.indices().getMapping(getMappingsRequest).mappings().get(index);
} catch (Exception e) {
throw new IndexException(String.format("Failed to get mapping of '%s' for type '%s'", name, mapping.typeAsString()), e);
}
try {
final ObjectNode newTypeMapping = mapper.valueToTree(typeMapping);
final ObjectNode currentTypeMapping = mapper.valueToTree(currentIndexMapping.getSourceAsMap());
SortedSet<String> compatibleChanges = Sets.newTreeSet();
SortedSet<String> incompatibleChanges = Sets.newTreeSet();
JsonDiff.diff(currentTypeMapping, newTypeMapping).forEach(change -> {
if (change.isAdd()) {
compatibleChanges.add(change.getFieldPath());
} else if (change.isMove() || change.isReplace()) {
incompatibleChanges.add(change.getFieldPath());
}
});
if (!incompatibleChanges.isEmpty()) {
log.warn("Cannot migrate index '{}' to new mapping with breaking changes on properties '{}'. Run repository reindex to migrate to new mapping schema or drop that index manually using the Elasticsearch API.", index, incompatibleChanges);
} else if (!compatibleChanges.isEmpty()) {
log.info("Applying mapping changes {} in index {}", compatibleChanges, index);
PutMappingRequest putMappingRequest = new PutMappingRequest(index).source(typeMapping);
AcknowledgedResponse response = client.indices().updateMapping(putMappingRequest);
checkState(response.isAcknowledged(), "Failed to update mapping '%s' for type '%s'", name, mapping.typeAsString());
// new fields do not require reindex, they will be added to new documents, existing documents don't have any data that needs reindex
if (hasFieldAliasChange(compatibleChanges)) {
if (bulkIndexByScroll(client, mapping, Expressions.matchAll(), "update", null, /*no script, in place update of docs to pick up mapping changes*/
"mapping migration")) {
mappingsToRefresh.add(mapping);
}
log.info("Migrated documents to new mapping in index '{}'", index);
}
}
} catch (IOException e) {
throw new IndexException(String.format("Failed to update mapping '%s' for type '%s'", name, mapping.typeAsString()), e);
}
} else {
// create index
final Map<String, Object> indexSettings;
try {
indexSettings = createIndexSettings();
log.info("Configuring '{}' index with settings: {}", index, indexSettings);
} catch (IOException e) {
throw new IndexException("Couldn't prepare settings for index " + index, e);
}
final CreateIndexRequest createIndexRequest = new CreateIndexRequest(index).mapping(typeMapping).settings(indexSettings);
try {
final CreateIndexResponse response = client.indices().create(createIndexRequest);
checkState(response.isAcknowledged(), "Failed to create index '%s' for type '%s'", name, mapping.typeAsString());
} catch (Exception e) {
throw new IndexException(String.format("Failed to create index '%s' for type '%s'", name, mapping.typeAsString()), e);
}
}
}
// wait until the cluster processes each index create request
waitForYellowHealth(indices());
if (!mappingsToRefresh.isEmpty()) {
refresh(mappingsToRefresh);
}
log.info("'{}' indexes are ready.", name);
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.CreateIndexRequest in project drill by apache.
the class ElasticComplexTypesTest method prepareData.
private static void prepareData() throws IOException {
restHighLevelClient = new RestHighLevelClient(RestClient.builder(HttpHost.create(HOST)));
String indexName = "arr";
indexNames.add(indexName);
CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("string_arr", Arrays.asList("a", "b", "c", "d"));
builder.field("int_arr", Arrays.asList(1, 2, 3, 4, 0));
builder.field("nest_int_arr", Arrays.asList(Arrays.asList(1, 2), Arrays.asList(3, 4, 0)));
builder.endObject();
IndexRequest indexRequest = new IndexRequest(indexName).source(builder);
restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);
restHighLevelClient.indices().refresh(new RefreshRequest(indexName), RequestOptions.DEFAULT);
indexName = "map";
indexNames.add(indexName);
createIndexRequest = new CreateIndexRequest(indexName);
restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT);
builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("prim_field", 321);
builder.field("nest_field", ImmutableMap.of("a", 123, "b", "abc"));
builder.field("more_nest_field", ImmutableMap.of("a", 123, "b", ImmutableMap.of("c", "abc")));
builder.field("map_arr", Collections.singletonList(ImmutableMap.of("a", 123, "b", ImmutableMap.of("c", "abc"))));
builder.endObject();
indexRequest = new IndexRequest(indexName).source(builder);
restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);
restHighLevelClient.indices().refresh(new RefreshRequest(indexName), RequestOptions.DEFAULT);
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.CreateIndexRequest in project drill by apache.
the class ElasticSearchPlanTest method prepareData.
private static void prepareData() throws IOException {
restHighLevelClient = new RestHighLevelClient(RestClient.builder(HttpHost.create(HOST)));
indexName = "nation";
CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("n_nationkey", 0);
builder.field("n_name", "ALGERIA");
builder.field("n_regionkey", 1);
builder.endObject();
IndexRequest indexRequest = new IndexRequest(indexName).source(builder);
restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);
restHighLevelClient.indices().refresh(new RefreshRequest(indexName), RequestOptions.DEFAULT);
}
Aggregations