Examples with IndexMetadata org.opensearch.cluster.metadata.IndexMetadata used on opensource projects

Example 11 with IndexMetadata

use of org.opensearch.cluster.metadata.IndexMetadata in project anomaly-detection by opensearch-project.

the class CustomIndexTests method testCorrectMapping.

public void testCorrectMapping() throws IOException {
    Map<String, Object> mappings = createMapping();
    IndexMetadata indexMetadata1 = new IndexMetadata.Builder(customIndexName).settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)).putMapping(new MappingMetadata("type1", Collections.singletonMap(CommonName.PROPERTIES, mappings))).build();
    when(clusterService.state()).thenReturn(ClusterState.builder(clusterName).metadata(Metadata.builder().put(indexMetadata1, true).build()).build());
    assertTrue(adIndices.isValidResultIndexMapping(customIndexName));
}

Example 12 with IndexMetadata

use of org.opensearch.cluster.metadata.IndexMetadata in project k-NN by opensearch-project.

the class IndexUtilTests method testGetLoadParameters.

public void testGetLoadParameters() {
    // Test faiss to ensure that space type gets set properly
    SpaceType spaceType1 = SpaceType.COSINESIMIL;
    KNNEngine knnEngine1 = KNNEngine.FAISS;
    String indexName = "my-test-index";
    Map<String, Object> loadParameters = getParametersAtLoading(spaceType1, knnEngine1, indexName);
    assertEquals(1, loadParameters.size());
    assertEquals(spaceType1.getValue(), loadParameters.get(SPACE_TYPE));
    // Test nmslib to ensure both space type and ef search are properly set
    SpaceType spaceType2 = SpaceType.L1;
    KNNEngine knnEngine2 = KNNEngine.NMSLIB;
    int efSearchValue = 413;
    // We use the constant for the setting here as opposed to the identifier of efSearch in nmslib jni
    Map<String, Object> indexSettings = ImmutableMap.of(KNN_ALGO_PARAM_EF_SEARCH, efSearchValue);
    // Because ef search comes from an index setting, we need to mock the long line of calls to get those
    // index settings
    Settings settings = Settings.builder().loadFromMap(indexSettings).build();
    IndexMetadata indexMetadata = mock(IndexMetadata.class);
    when(indexMetadata.getSettings()).thenReturn(settings);
    Metadata metadata = mock(Metadata.class);
    when(metadata.index(anyString())).thenReturn(indexMetadata);
    ClusterState clusterState = mock(ClusterState.class);
    when(clusterState.getMetadata()).thenReturn(metadata);
    ClusterService clusterService = mock(ClusterService.class);
    when(clusterService.state()).thenReturn(clusterState);
    KNNSettings.state().setClusterService(clusterService);
    loadParameters = getParametersAtLoading(spaceType2, knnEngine2, indexName);
    assertEquals(2, loadParameters.size());
    assertEquals(spaceType2.getValue(), loadParameters.get(SPACE_TYPE));
    assertEquals(efSearchValue, loadParameters.get(HNSW_ALGO_EF_SEARCH));
}

Example 13 with IndexMetadata

use of org.opensearch.cluster.metadata.IndexMetadata in project k-NN by opensearch-project.

the class VectorReader method read.

/**
 * Read vectors from a provided index/field and pass them to vectorConsumer that will do something with them.
 *
 * @param clusterService cluster service to get information about the index
 * @param indexName name of index containing vectors
 * @param fieldName name of field containing vectors
 * @param maxVectorCount maximum number of vectors to return
 * @param searchSize maximum number of vectors to return in a given search
 * @param vectorConsumer consumer used to do something with the collected vectors after each search
 * @param listener ActionListener that should be called once all search operations complete
 */
public void read(ClusterService clusterService, String indexName, String fieldName, int maxVectorCount, int searchSize, Consumer<List<Float[]>> vectorConsumer, ActionListener<SearchResponse> listener) {
    ValidationException validationException = null;
    // Validate arguments
    if (maxVectorCount <= 0) {
        validationException = new ValidationException();
        validationException.addValidationError("maxVectorCount must be >= 0");
    }
    if (searchSize > 10000 || searchSize <= 0) {
        validationException = validationException == null ? new ValidationException() : validationException;
        validationException.addValidationError("searchSize must be > 0 and <= 10000");
    }
    IndexMetadata indexMetadata = clusterService.state().metadata().index(indexName);
    if (indexMetadata == null) {
        validationException = validationException == null ? new ValidationException() : validationException;
        validationException.addValidationError("index \"" + indexName + "\" does not exist");
        throw validationException;
    }
    ValidationException fieldValidationException = IndexUtil.validateKnnField(indexMetadata, fieldName, -1, null);
    if (fieldValidationException != null) {
        validationException = validationException == null ? new ValidationException() : validationException;
        validationException.addValidationErrors(validationException.validationErrors());
    }
    if (validationException != null) {
        throw validationException;
    }
    // Start reading vectors from index
    SearchScrollRequestBuilder searchScrollRequestBuilder = createSearchScrollRequestBuilder();
    ActionListener<SearchResponse> vectorReaderListener = new VectorReaderListener(client, fieldName, maxVectorCount, 0, listener, vectorConsumer, searchScrollRequestBuilder);
    createSearchRequestBuilder(indexName, fieldName, Integer.min(maxVectorCount, searchSize)).execute(vectorReaderListener);
}

Example 14 with IndexMetadata

use of org.opensearch.cluster.metadata.IndexMetadata in project k-NN by opensearch-project.

the class TrainingModelRequest method validate.

@Override
public ActionRequestValidationException validate() {
    ActionRequestValidationException exception = null;
    // Check if model id exists via model metadata
    if (modelDao.getMetadata(modelId) != null) {
        exception = new ActionRequestValidationException();
        exception.addValidationError("Model with id=\"" + modelId + "\" already exists");
        return exception;
    }
    // Confirm that the passed in knnMethodContext is valid and requires training
    ValidationException validationException = this.knnMethodContext.validate();
    if (validationException != null) {
        exception = new ActionRequestValidationException();
        exception.addValidationErrors(validationException.validationErrors());
    }
    if (!this.knnMethodContext.isTrainingRequired()) {
        exception = exception == null ? new ActionRequestValidationException() : exception;
        exception.addValidationError("Method does not require training.");
    }
    // Check if preferred node is real
    if (preferredNodeId != null && !clusterService.state().nodes().getDataNodes().containsKey(preferredNodeId)) {
        exception = exception == null ? new ActionRequestValidationException() : exception;
        exception.addValidationError("Preferred node \"" + preferredNodeId + "\" does not exist");
    }
    // Check if description is too long
    if (description != null && description.length() > KNNConstants.MAX_MODEL_DESCRIPTION_LENGTH) {
        exception = exception == null ? new ActionRequestValidationException() : exception;
        exception.addValidationError("Description exceeds limit of " + KNNConstants.MAX_MODEL_DESCRIPTION_LENGTH + " characters");
    }
    // Validate training index exists
    IndexMetadata indexMetadata = clusterService.state().metadata().index(trainingIndex);
    if (indexMetadata == null) {
        exception = exception == null ? new ActionRequestValidationException() : exception;
        exception.addValidationError("Index \"" + this.trainingIndex + "\" does not exist.");
        return exception;
    }
    // Validate the training field
    ValidationException fieldValidation = IndexUtil.validateKnnField(indexMetadata, this.trainingField, this.dimension, modelDao);
    if (fieldValidation != null) {
        exception = exception == null ? new ActionRequestValidationException() : exception;
        exception.addValidationErrors(fieldValidation.validationErrors());
    }
    return exception;
}

Example 15 with IndexMetadata

use of org.opensearch.cluster.metadata.IndexMetadata in project anomaly-detection by opensearch-project.

the class IndexAnomalyDetectorTransportActionTests method setUp.

@SuppressWarnings("unchecked")
@Override
@Before
public void setUp() throws Exception {
    super.setUp();
    clusterService = mock(ClusterService.class);
    clusterSettings = new ClusterSettings(Settings.EMPTY, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(AnomalyDetectorSettings.FILTER_BY_BACKEND_ROLES))));
    when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
    ClusterName clusterName = new ClusterName("test");
    Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
    final Settings.Builder existingSettings = Settings.builder().put(indexSettings).put(IndexMetadata.SETTING_INDEX_UUID, "test2UUID");
    IndexMetadata indexMetaData = IndexMetadata.builder(AnomalyDetector.ANOMALY_DETECTORS_INDEX).settings(existingSettings).build();
    final ImmutableOpenMap<String, IndexMetadata> indices = ImmutableOpenMap.<String, IndexMetadata>builder().fPut(AnomalyDetector.ANOMALY_DETECTORS_INDEX, indexMetaData).build();
    ClusterState clusterState = ClusterState.builder(clusterName).metadata(Metadata.builder().indices(indices).build()).build();
    when(clusterService.state()).thenReturn(clusterState);
    adTaskManager = mock(ADTaskManager.class);
    searchFeatureDao = mock(SearchFeatureDao.class);
    action = new IndexAnomalyDetectorTransportAction(mock(TransportService.class), mock(ActionFilters.class), client(), clusterService, indexSettings(), mock(AnomalyDetectionIndices.class), xContentRegistry(), adTaskManager, searchFeatureDao);
    task = mock(Task.class);
    AnomalyDetector detector = TestHelpers.randomAnomalyDetector(ImmutableMap.of("testKey", "testValue"), Instant.now());
    GetResponse getDetectorResponse = TestHelpers.createGetResponse(detector, detector.getDetectorId(), AnomalyDetector.ANOMALY_DETECTORS_INDEX);
    doAnswer(invocation -> {
        Object[] args = invocation.getArguments();
        assertTrue(String.format("The size of args is %d.  Its content is %s", args.length, Arrays.toString(args)), args.length == 2);
        assertTrue(args[0] instanceof GetRequest);
        assertTrue(args[1] instanceof ActionListener);
        ActionListener<GetResponse> listener = (ActionListener<GetResponse>) args[1];
        listener.onResponse(getDetectorResponse);
        return null;
    }).when(client).get(any(GetRequest.class), any());
    SearchHits hits = new SearchHits(new SearchHit[] {}, null, Float.NaN);
    SearchResponseSections searchSections = new SearchResponseSections(hits, null, null, false, false, null, 1);
    SearchResponse searchResponse = new SearchResponse(searchSections, null, 1, 1, 0, 30, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
    doAnswer(invocation -> {
        Object[] args = invocation.getArguments();
        assertTrue(String.format("The size of args is %d.  Its content is %s", args.length, Arrays.toString(args)), args.length == 2);
        assertTrue(args[0] instanceof SearchRequest);
        assertTrue(args[1] instanceof ActionListener);
        ActionListener<SearchResponse> listener = (ActionListener<SearchResponse>) args[1];
        listener.onResponse(searchResponse);
        return null;
    }).when(client).search(any(SearchRequest.class), any());
    request = new IndexAnomalyDetectorRequest("1234", 4567, 7890, WriteRequest.RefreshPolicy.IMMEDIATE, detector, RestRequest.Method.PUT, TimeValue.timeValueSeconds(60), 1000, 10, 5);
    response = new ActionListener<IndexAnomalyDetectorResponse>() {

        @Override
        public void onResponse(IndexAnomalyDetectorResponse indexResponse) {
            // onResponse will not be called as we do not have the AD index
            Assert.assertTrue(false);
        }

        @Override
        public void onFailure(Exception e) {
            Assert.assertTrue(true);
        }
    };
}