Search in sources :

Example 1 with KNNMethodContext

use of org.opensearch.knn.index.KNNMethodContext in project k-NN by opensearch-project.

the class TrainingModelRequestTests method testValidation_invalid_trainingIndexDoesNotExist.

public void testValidation_invalid_trainingIndexDoesNotExist() {
    // Check that validation produces exception when the training index doesnt exist
    // Setup the training request
    String modelId = "test-model-id";
    KNNMethodContext knnMethodContext = mock(KNNMethodContext.class);
    when(knnMethodContext.validate()).thenReturn(null);
    when(knnMethodContext.isTrainingRequired()).thenReturn(true);
    int dimension = 10;
    String trainingIndex = "test-training-index";
    String trainingField = "test-training-field";
    TrainingModelRequest trainingModelRequest = new TrainingModelRequest(modelId, knnMethodContext, dimension, trainingIndex, trainingField, null, null);
    // Mock the model dao to return null so that no exception is produced
    ModelDao modelDao = mock(ModelDao.class);
    when(modelDao.getMetadata(modelId)).thenReturn(null);
    Metadata metadata = mock(Metadata.class);
    when(metadata.index(trainingIndex)).thenReturn(null);
    ClusterState clusterState = mock(ClusterState.class);
    when(clusterState.metadata()).thenReturn(metadata);
    ClusterService clusterService = mock(ClusterService.class);
    when(clusterService.state()).thenReturn(clusterState);
    // Initialize static components with the mocks
    TrainingModelRequest.initialize(modelDao, clusterService);
    // Test that validation produces model already exists error message
    ActionRequestValidationException exception = trainingModelRequest.validate();
    assertNotNull(exception);
    List<String> validationErrors = exception.validationErrors();
    assertEquals(1, validationErrors.size());
    assertTrue(validationErrors.get(0).contains("does not exist"));
}
Also used : ClusterState(org.opensearch.cluster.ClusterState) KNNMethodContext(org.opensearch.knn.index.KNNMethodContext) ClusterService(org.opensearch.cluster.service.ClusterService) ActionRequestValidationException(org.opensearch.action.ActionRequestValidationException) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) Metadata(org.opensearch.cluster.metadata.Metadata) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ModelMetadata(org.opensearch.knn.indices.ModelMetadata) ModelDao(org.opensearch.knn.indices.ModelDao)

Example 2 with KNNMethodContext

use of org.opensearch.knn.index.KNNMethodContext in project k-NN by opensearch-project.

the class TrainingModelRequestTests method testValidation_invalid_descriptionToLong.

public void testValidation_invalid_descriptionToLong() {
    // Setup the training request
    String modelId = "test-model-id";
    KNNMethodContext knnMethodContext = mock(KNNMethodContext.class);
    when(knnMethodContext.validate()).thenReturn(null);
    when(knnMethodContext.isTrainingRequired()).thenReturn(true);
    int dimension = 10;
    String trainingIndex = "test-training-index";
    String trainingField = "test-training-field";
    String trainingFieldModeId = "training-field-model-id";
    char[] chars = new char[KNNConstants.MAX_MODEL_DESCRIPTION_LENGTH + 1];
    Arrays.fill(chars, 'a');
    String description = new String(chars);
    TrainingModelRequest trainingModelRequest = new TrainingModelRequest(modelId, knnMethodContext, dimension, trainingIndex, trainingField, null, description);
    // Mock the model dao to return metadata for modelId to recognize it is a duplicate
    ModelMetadata trainingFieldModelMetadata = mock(ModelMetadata.class);
    when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension);
    ModelDao modelDao = mock(ModelDao.class);
    when(modelDao.getMetadata(modelId)).thenReturn(null);
    when(modelDao.getMetadata(trainingFieldModeId)).thenReturn(trainingFieldModelMetadata);
    // Cluster service that wont produce validation exception
    ClusterService clusterService = getClusterServiceForValidReturns(trainingIndex, trainingField, dimension);
    // Initialize static components with the mocks
    TrainingModelRequest.initialize(modelDao, clusterService);
    // Test that validation produces model already exists error message
    ActionRequestValidationException exception = trainingModelRequest.validate();
    assertNotNull(exception);
    List<String> validationErrors = exception.validationErrors();
    assertEquals(1, validationErrors.size());
    assertTrue(validationErrors.get(0).contains("Description exceeds limit"));
}
Also used : KNNMethodContext(org.opensearch.knn.index.KNNMethodContext) ClusterService(org.opensearch.cluster.service.ClusterService) ActionRequestValidationException(org.opensearch.action.ActionRequestValidationException) ModelMetadata(org.opensearch.knn.indices.ModelMetadata) ModelDao(org.opensearch.knn.indices.ModelDao)

Example 3 with KNNMethodContext

use of org.opensearch.knn.index.KNNMethodContext in project k-NN by opensearch-project.

the class TrainingModelRequestTests method testValidation_invalid_dimensionDoesNotMatch.

public void testValidation_invalid_dimensionDoesNotMatch() {
    // Check that validation produces exception when dimension does not match
    // Setup the training request
    String modelId = "test-model-id";
    KNNMethodContext knnMethodContext = mock(KNNMethodContext.class);
    when(knnMethodContext.validate()).thenReturn(null);
    when(knnMethodContext.isTrainingRequired()).thenReturn(true);
    int dimension = 10;
    String trainingIndex = "test-training-index";
    String trainingField = "test-training-field";
    TrainingModelRequest trainingModelRequest = new TrainingModelRequest(modelId, knnMethodContext, dimension, trainingIndex, trainingField, null, null);
    // Mock the model dao to return null so that no exception is produced
    ModelDao modelDao = mock(ModelDao.class);
    when(modelDao.getMetadata(modelId)).thenReturn(null);
    // Return  mapping with different dimension
    Map<String, Object> mappingMap = ImmutableMap.of("properties", ImmutableMap.of(trainingField, ImmutableMap.of("type", KNNVectorFieldMapper.CONTENT_TYPE, KNNConstants.DIMENSION, dimension + 1)));
    MappingMetadata mappingMetadata = mock(MappingMetadata.class);
    when(mappingMetadata.getSourceAsMap()).thenReturn(mappingMap);
    IndexMetadata indexMetadata = mock(IndexMetadata.class);
    when(indexMetadata.mapping()).thenReturn(mappingMetadata);
    Metadata metadata = mock(Metadata.class);
    when(metadata.index(trainingIndex)).thenReturn(indexMetadata);
    ClusterState clusterState = mock(ClusterState.class);
    when(clusterState.metadata()).thenReturn(metadata);
    ClusterService clusterService = mock(ClusterService.class);
    when(clusterService.state()).thenReturn(clusterState);
    // Initialize static components with the mocks
    TrainingModelRequest.initialize(modelDao, clusterService);
    // Test that validation produces model already exists error message
    ActionRequestValidationException exception = trainingModelRequest.validate();
    assertNotNull(exception);
    List<String> validationErrors = exception.validationErrors();
    assertEquals(1, validationErrors.size());
    assertTrue(validationErrors.get(0).contains("different from dimension"));
}
Also used : ClusterState(org.opensearch.cluster.ClusterState) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) Metadata(org.opensearch.cluster.metadata.Metadata) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ModelMetadata(org.opensearch.knn.indices.ModelMetadata) KNNMethodContext(org.opensearch.knn.index.KNNMethodContext) ClusterService(org.opensearch.cluster.service.ClusterService) ActionRequestValidationException(org.opensearch.action.ActionRequestValidationException) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ModelDao(org.opensearch.knn.indices.ModelDao) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata)

Example 4 with KNNMethodContext

use of org.opensearch.knn.index.KNNMethodContext in project k-NN by opensearch-project.

the class TrainingModelRequestTests method testValidation_invalid_trainingFieldDoesNotExist.

public void testValidation_invalid_trainingFieldDoesNotExist() {
    // Check that validation produces exception when the training field doesnt exist
    // Setup the training request
    String modelId = "test-model-id";
    KNNMethodContext knnMethodContext = mock(KNNMethodContext.class);
    when(knnMethodContext.validate()).thenReturn(null);
    when(knnMethodContext.isTrainingRequired()).thenReturn(true);
    int dimension = 10;
    String trainingIndex = "test-training-index";
    String trainingField = "test-training-field";
    TrainingModelRequest trainingModelRequest = new TrainingModelRequest(modelId, knnMethodContext, dimension, trainingIndex, trainingField, null, null);
    // Mock the model dao to return null so that no exception is produced
    ModelDao modelDao = mock(ModelDao.class);
    when(modelDao.getMetadata(modelId)).thenReturn(null);
    // Return empty mapping so that training field does not exist
    MappingMetadata mappingMetadata = mock(MappingMetadata.class);
    when(mappingMetadata.getSourceAsMap()).thenReturn(Collections.emptyMap());
    IndexMetadata indexMetadata = mock(IndexMetadata.class);
    when(indexMetadata.mapping()).thenReturn(mappingMetadata);
    Metadata metadata = mock(Metadata.class);
    when(metadata.index(trainingIndex)).thenReturn(indexMetadata);
    ClusterState clusterState = mock(ClusterState.class);
    when(clusterState.metadata()).thenReturn(metadata);
    ClusterService clusterService = mock(ClusterService.class);
    when(clusterService.state()).thenReturn(clusterState);
    // Initialize static components with the mocks
    TrainingModelRequest.initialize(modelDao, clusterService);
    // Test that validation produces model already exists error message
    ActionRequestValidationException exception = trainingModelRequest.validate();
    assertNotNull(exception);
    List<String> validationErrors = exception.validationErrors();
    assertEquals(1, validationErrors.size());
    assertTrue(validationErrors.get(0).contains("does not exist"));
}
Also used : ClusterState(org.opensearch.cluster.ClusterState) KNNMethodContext(org.opensearch.knn.index.KNNMethodContext) ClusterService(org.opensearch.cluster.service.ClusterService) ActionRequestValidationException(org.opensearch.action.ActionRequestValidationException) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) Metadata(org.opensearch.cluster.metadata.Metadata) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ModelMetadata(org.opensearch.knn.indices.ModelMetadata) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ModelDao(org.opensearch.knn.indices.ModelDao) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata)

Example 5 with KNNMethodContext

use of org.opensearch.knn.index.KNNMethodContext in project k-NN by opensearch-project.

the class TrainingModelRequestTests method testValidation_invalid_preferredNodeDoesNotExist.

public void testValidation_invalid_preferredNodeDoesNotExist() {
    // Check that validation produces exception preferred node does not exist
    // Setup the training request
    String modelId = "test-model-id";
    KNNMethodContext knnMethodContext = mock(KNNMethodContext.class);
    when(knnMethodContext.validate()).thenReturn(null);
    when(knnMethodContext.isTrainingRequired()).thenReturn(true);
    int dimension = 10;
    String trainingIndex = "test-training-index";
    String trainingField = "test-training-field";
    String preferredNode = "preferred-node";
    TrainingModelRequest trainingModelRequest = new TrainingModelRequest(modelId, knnMethodContext, dimension, trainingIndex, trainingField, preferredNode, null);
    // Mock the model dao to return metadata for modelId to recognize it is a duplicate
    ModelDao modelDao = mock(ModelDao.class);
    when(modelDao.getMetadata(modelId)).thenReturn(null);
    // This cluster service mocking should not produce exception
    Map<String, Object> mappingMap = ImmutableMap.of("properties", ImmutableMap.of(trainingField, ImmutableMap.of("type", KNNVectorFieldMapper.CONTENT_TYPE, KNNConstants.DIMENSION, dimension)));
    MappingMetadata mappingMetadata = mock(MappingMetadata.class);
    when(mappingMetadata.getSourceAsMap()).thenReturn(mappingMap);
    IndexMetadata indexMetadata = mock(IndexMetadata.class);
    when(indexMetadata.mapping()).thenReturn(mappingMetadata);
    Metadata metadata = mock(Metadata.class);
    when(metadata.index(trainingIndex)).thenReturn(indexMetadata);
    // Empty set of data nodes to produce exception
    DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class);
    when(discoveryNodes.getDataNodes()).thenReturn(ImmutableOpenMap.of());
    ClusterState clusterState = mock(ClusterState.class);
    when(clusterState.metadata()).thenReturn(metadata);
    when(clusterState.nodes()).thenReturn(discoveryNodes);
    ClusterService clusterService = mock(ClusterService.class);
    when(clusterService.state()).thenReturn(clusterState);
    // Initialize static components with the mocks
    TrainingModelRequest.initialize(modelDao, clusterService);
    // Test that validation produces model already exists error message
    ActionRequestValidationException exception = trainingModelRequest.validate();
    assertNotNull(exception);
    List<String> validationErrors = exception.validationErrors();
    assertEquals(1, validationErrors.size());
    assertTrue(validationErrors.get(0).contains("Preferred node"));
}
Also used : ClusterState(org.opensearch.cluster.ClusterState) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) Metadata(org.opensearch.cluster.metadata.Metadata) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ModelMetadata(org.opensearch.knn.indices.ModelMetadata) KNNMethodContext(org.opensearch.knn.index.KNNMethodContext) ClusterService(org.opensearch.cluster.service.ClusterService) ActionRequestValidationException(org.opensearch.action.ActionRequestValidationException) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ModelDao(org.opensearch.knn.indices.ModelDao) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes)

Aggregations

KNNMethodContext (org.opensearch.knn.index.KNNMethodContext)24 ActionRequestValidationException (org.opensearch.action.ActionRequestValidationException)10 ClusterService (org.opensearch.cluster.service.ClusterService)10 ModelDao (org.opensearch.knn.indices.ModelDao)10 ModelMetadata (org.opensearch.knn.indices.ModelMetadata)9 MethodComponentContext (org.opensearch.knn.index.MethodComponentContext)7 NativeMemoryCacheManager (org.opensearch.knn.index.memory.NativeMemoryCacheManager)7 ClusterState (org.opensearch.cluster.ClusterState)6 IndexMetadata (org.opensearch.cluster.metadata.IndexMetadata)6 MappingMetadata (org.opensearch.cluster.metadata.MappingMetadata)6 Metadata (org.opensearch.cluster.metadata.Metadata)6 KNNEngine (org.opensearch.knn.index.util.KNNEngine)6 Model (org.opensearch.knn.indices.Model)6 NativeMemoryAllocation (org.opensearch.knn.index.memory.NativeMemoryAllocation)5 NativeMemoryEntryContext (org.opensearch.knn.index.memory.NativeMemoryEntryContext)5 SpaceType (org.opensearch.knn.index.SpaceType)3 Path (java.nio.file.Path)2 DiscoveryNodes (org.opensearch.cluster.node.DiscoveryNodes)2 ValidationException (org.opensearch.common.ValidationException)2 XContentBuilder (org.opensearch.common.xcontent.XContentBuilder)2