Examples with IngestService org.opensearch.ingest.IngestService used on opensource projects

Example 6 with IngestService

use of org.opensearch.ingest.IngestService in project OpenSearch by opensearch-project.

the class TransportBulkActionIngestTests method testSingleItemBulkActionIngestLocal.

public void testSingleItemBulkActionIngestLocal() throws Exception {
    Exception exception = new Exception("fake exception");
    IndexRequest indexRequest = new IndexRequest("index").id("id");
    indexRequest.source(emptyMap());
    indexRequest.setPipeline("testpipeline");
    AtomicBoolean responseCalled = new AtomicBoolean(false);
    AtomicBoolean failureCalled = new AtomicBoolean(false);
    singleItemBulkWriteAction.execute(null, indexRequest, ActionListener.wrap(response -> {
        responseCalled.set(true);
    }, e -> {
        assertThat(e, sameInstance(exception));
        failureCalled.set(true);
    }));
    // check failure works, and passes through to the listener
    // haven't executed yet
    assertFalse(action.isExecuted);
    assertFalse(responseCalled.get());
    assertFalse(failureCalled.get());
    verify(ingestService).executeBulkRequest(eq(1), bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture(), any(), eq(Names.WRITE));
    completionHandler.getValue().accept(null, exception);
    assertTrue(failureCalled.get());
    // now check success
    // this is done by the real pipeline execution service when processing
    indexRequest.setPipeline(IngestService.NOOP_PIPELINE_NAME);
    completionHandler.getValue().accept(DUMMY_WRITE_THREAD, null);
    assertTrue(action.isExecuted);
    // listener would only be called by real index action, not our mocked one
    assertFalse(responseCalled.get());
    verifyNoInteractions(transportService);
}

Example 7 with IngestService

use of org.opensearch.ingest.IngestService in project OpenSearch by opensearch-project.

the class TransportBulkActionIngestTests method testFindDefaultPipelineFromTemplateMatch.

public void testFindDefaultPipelineFromTemplateMatch() {
    Exception exception = new Exception("fake exception");
    ClusterState state = clusterService.state();
    ImmutableOpenMap.Builder<String, IndexTemplateMetadata> templateMetadataBuilder = ImmutableOpenMap.builder();
    templateMetadataBuilder.put("template1", IndexTemplateMetadata.builder("template1").patterns(Arrays.asList("missing_index")).order(1).settings(Settings.builder().put(IndexSettings.DEFAULT_PIPELINE.getKey(), "pipeline1").build()).build());
    templateMetadataBuilder.put("template2", IndexTemplateMetadata.builder("template2").patterns(Arrays.asList("missing_*")).order(2).settings(Settings.builder().put(IndexSettings.DEFAULT_PIPELINE.getKey(), "pipeline2").build()).build());
    templateMetadataBuilder.put("template3", IndexTemplateMetadata.builder("template3").patterns(Arrays.asList("missing*")).order(3).build());
    templateMetadataBuilder.put("template4", IndexTemplateMetadata.builder("template4").patterns(Arrays.asList("nope")).order(4).settings(Settings.builder().put(IndexSettings.DEFAULT_PIPELINE.getKey(), "pipeline4").build()).build());
    Metadata metadata = mock(Metadata.class);
    when(state.metadata()).thenReturn(metadata);
    when(state.getMetadata()).thenReturn(metadata);
    when(metadata.templates()).thenReturn(templateMetadataBuilder.build());
    when(metadata.getTemplates()).thenReturn(templateMetadataBuilder.build());
    when(metadata.indices()).thenReturn(ImmutableOpenMap.of());
    IndexRequest indexRequest = new IndexRequest("missing_index").id("id");
    indexRequest.source(emptyMap());
    AtomicBoolean responseCalled = new AtomicBoolean(false);
    AtomicBoolean failureCalled = new AtomicBoolean(false);
    singleItemBulkWriteAction.execute(null, indexRequest, ActionListener.wrap(response -> responseCalled.set(true), e -> {
        assertThat(e, sameInstance(exception));
        failureCalled.set(true);
    }));
    assertEquals("pipeline2", indexRequest.getPipeline());
    verify(ingestService).executeBulkRequest(eq(1), bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture(), any(), eq(Names.WRITE));
}

Example 8 with IngestService

use of org.opensearch.ingest.IngestService in project OpenSearch by opensearch-project.

the class TransportBulkActionIngestTests method testDoExecuteCalledTwiceCorrectly.

public void testDoExecuteCalledTwiceCorrectly() throws Exception {
    Exception exception = new Exception("fake exception");
    IndexRequest indexRequest = new IndexRequest("missing_index").id("id");
    indexRequest.setPipeline("testpipeline");
    indexRequest.source(emptyMap());
    AtomicBoolean responseCalled = new AtomicBoolean(false);
    AtomicBoolean failureCalled = new AtomicBoolean(false);
    action.needToCheck = true;
    action.indexCreated = false;
    singleItemBulkWriteAction.execute(null, indexRequest, ActionListener.wrap(response -> responseCalled.set(true), e -> {
        assertThat(e, sameInstance(exception));
        failureCalled.set(true);
    }));
    // check failure works, and passes through to the listener
    // haven't executed yet
    assertFalse(action.isExecuted);
    // no index yet
    assertFalse(action.indexCreated);
    assertFalse(responseCalled.get());
    assertFalse(failureCalled.get());
    verify(ingestService).executeBulkRequest(eq(1), bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture(), any(), eq(Names.WRITE));
    completionHandler.getValue().accept(null, exception);
    // still no index yet, the ingest node failed.
    assertFalse(action.indexCreated);
    assertTrue(failureCalled.get());
    // now check success
    // this is done by the real pipeline execution service when processing
    indexRequest.setPipeline(IngestService.NOOP_PIPELINE_NAME);
    completionHandler.getValue().accept(DUMMY_WRITE_THREAD, null);
    assertTrue(action.isExecuted);
    // now the index is created since we skipped the ingest node path.
    assertTrue(action.indexCreated);
    // listener would only be called by real index action, not our mocked one
    assertFalse(responseCalled.get());
    verifyNoInteractions(transportService);
}

Example 9 with IngestService

use of org.opensearch.ingest.IngestService in project OpenSearch by opensearch-project.

the class TransportBulkActionIngestTests method validatePipelineWithBulkUpsert.

private void validatePipelineWithBulkUpsert(@Nullable String indexRequestIndexName, String updateRequestIndexName) throws Exception {
    Exception exception = new Exception("fake exception");
    BulkRequest bulkRequest = new BulkRequest();
    IndexRequest indexRequest1 = new IndexRequest(indexRequestIndexName).id("id1").source(emptyMap());
    IndexRequest indexRequest2 = new IndexRequest(indexRequestIndexName).id("id2").source(emptyMap());
    IndexRequest indexRequest3 = new IndexRequest(indexRequestIndexName).id("id3").source(emptyMap());
    UpdateRequest upsertRequest = new UpdateRequest(updateRequestIndexName, "id1").upsert(indexRequest1).script(mockScript("1"));
    UpdateRequest docAsUpsertRequest = new UpdateRequest(updateRequestIndexName, "id2").doc(indexRequest2).docAsUpsert(true);
    // this test only covers the mechanics that scripted bulk upserts will execute a default pipeline. However, in practice scripted
    // bulk upserts with a default pipeline are a bit surprising since the script executes AFTER the pipeline.
    UpdateRequest scriptedUpsert = new UpdateRequest(updateRequestIndexName, "id2").upsert(indexRequest3).script(mockScript("1")).scriptedUpsert(true);
    bulkRequest.add(upsertRequest).add(docAsUpsertRequest).add(scriptedUpsert);
    AtomicBoolean responseCalled = new AtomicBoolean(false);
    AtomicBoolean failureCalled = new AtomicBoolean(false);
    assertNull(indexRequest1.getPipeline());
    assertNull(indexRequest2.getPipeline());
    assertNull(indexRequest3.getPipeline());
    action.execute(null, bulkRequest, ActionListener.wrap(response -> {
        BulkItemResponse itemResponse = response.iterator().next();
        assertThat(itemResponse.getFailure().getMessage(), containsString("fake exception"));
        responseCalled.set(true);
    }, e -> {
        assertThat(e, sameInstance(exception));
        failureCalled.set(true);
    }));
    // check failure works, and passes through to the listener
    // haven't executed yet
    assertFalse(action.isExecuted);
    assertFalse(responseCalled.get());
    assertFalse(failureCalled.get());
    verify(ingestService).executeBulkRequest(eq(bulkRequest.numberOfActions()), bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture(), any(), eq(Names.WRITE));
    assertEquals(indexRequest1.getPipeline(), "default_pipeline");
    assertEquals(indexRequest2.getPipeline(), "default_pipeline");
    assertEquals(indexRequest3.getPipeline(), "default_pipeline");
    completionHandler.getValue().accept(null, exception);
    assertTrue(failureCalled.get());
    // now check success of the transport bulk action
    // this is done by the real pipeline execution service when processing
    indexRequest1.setPipeline(IngestService.NOOP_PIPELINE_NAME);
    // this is done by the real pipeline execution service when processing
    indexRequest2.setPipeline(IngestService.NOOP_PIPELINE_NAME);
    // this is done by the real pipeline execution service when processing
    indexRequest3.setPipeline(IngestService.NOOP_PIPELINE_NAME);
    completionHandler.getValue().accept(DUMMY_WRITE_THREAD, null);
    assertTrue(action.isExecuted);
    // listener would only be called by real index action, not our mocked one
    assertFalse(responseCalled.get());
    verifyNoInteractions(transportService);
}

Example 10 with IngestService

use of org.opensearch.ingest.IngestService in project OpenSearch by opensearch-project.

the class TransportBulkActionIngestTests method testSingleItemBulkActionIngestSkipped.

public void testSingleItemBulkActionIngestSkipped() throws Exception {
    IndexRequest indexRequest = new IndexRequest("index").id("id");
    indexRequest.source(emptyMap());
    singleItemBulkWriteAction.execute(null, indexRequest, ActionListener.wrap(response -> {
    }, exception -> {
        throw new AssertionError(exception);
    }));
    assertTrue(action.isExecuted);
    verifyNoInteractions(ingestService);
}