Examples with GetResponse org.elasticsearch.action.get.GetResponse used on opensource projects

Example 61 with GetResponse

use of org.elasticsearch.action.get.GetResponse in project elasticsearch by elastic.

the class UpdateIT method testStressUpdateDeleteConcurrency.

public void testStressUpdateDeleteConcurrency() throws Exception {
    //We create an index with merging disabled so that deletes don't get merged away
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)));
    ensureGreen();
    final int numberOfThreads = scaledRandomIntBetween(3, 5);
    final int numberOfIdsPerThread = scaledRandomIntBetween(3, 10);
    final int numberOfUpdatesPerId = scaledRandomIntBetween(10, 100);
    final int retryOnConflict = randomIntBetween(0, 1);
    final CountDownLatch latch = new CountDownLatch(numberOfThreads);
    final CountDownLatch startLatch = new CountDownLatch(1);
    final List<Throwable> failures = new CopyOnWriteArrayList<>();
    final class UpdateThread extends Thread {

        final Map<Integer, Integer> failedMap = new HashMap<>();

        final int numberOfIds;

        final int updatesPerId;

        final int maxUpdateRequests = numberOfIdsPerThread * numberOfUpdatesPerId;

        final int maxDeleteRequests = numberOfIdsPerThread * numberOfUpdatesPerId;

        private final Semaphore updateRequestsOutstanding = new Semaphore(maxUpdateRequests);

        private final Semaphore deleteRequestsOutstanding = new Semaphore(maxDeleteRequests);

        UpdateThread(int numberOfIds, int updatesPerId) {
            this.numberOfIds = numberOfIds;
            this.updatesPerId = updatesPerId;
        }

        final class UpdateListener implements ActionListener<UpdateResponse> {

            int id;

            UpdateListener(int id) {
                this.id = id;
            }

            @Override
            public void onResponse(UpdateResponse updateResponse) {
                updateRequestsOutstanding.release(1);
            }

            @Override
            public void onFailure(Exception e) {
                synchronized (failedMap) {
                    incrementMapValue(id, failedMap);
                }
                updateRequestsOutstanding.release(1);
            }
        }

        final class DeleteListener implements ActionListener<DeleteResponse> {

            int id;

            DeleteListener(int id) {
                this.id = id;
            }

            @Override
            public void onResponse(DeleteResponse deleteResponse) {
                deleteRequestsOutstanding.release(1);
            }

            @Override
            public void onFailure(Exception e) {
                synchronized (failedMap) {
                    incrementMapValue(id, failedMap);
                }
                deleteRequestsOutstanding.release(1);
            }
        }

        @Override
        public void run() {
            try {
                startLatch.await();
                boolean hasWaitedForNoNode = false;
                for (int j = 0; j < numberOfIds; j++) {
                    for (int k = 0; k < numberOfUpdatesPerId; ++k) {
                        updateRequestsOutstanding.acquire();
                        try {
                            UpdateRequest ur = client().prepareUpdate("test", "type1", Integer.toString(j)).setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap())).setRetryOnConflict(retryOnConflict).setUpsert(jsonBuilder().startObject().field("field", 1).endObject()).request();
                            client().update(ur, new UpdateListener(j));
                        } catch (NoNodeAvailableException nne) {
                            updateRequestsOutstanding.release();
                            synchronized (failedMap) {
                                incrementMapValue(j, failedMap);
                            }
                            if (hasWaitedForNoNode) {
                                throw nne;
                            }
                            logger.warn("Got NoNodeException waiting for 1 second for things to recover.");
                            hasWaitedForNoNode = true;
                            Thread.sleep(1000);
                        }
                        try {
                            deleteRequestsOutstanding.acquire();
                            DeleteRequest dr = client().prepareDelete("test", "type1", Integer.toString(j)).request();
                            client().delete(dr, new DeleteListener(j));
                        } catch (NoNodeAvailableException nne) {
                            deleteRequestsOutstanding.release();
                            synchronized (failedMap) {
                                incrementMapValue(j, failedMap);
                            }
                            if (hasWaitedForNoNode) {
                                throw nne;
                            }
                            logger.warn("Got NoNodeException waiting for 1 second for things to recover.");
                            hasWaitedForNoNode = true;
                            //Wait for no-node to clear
                            Thread.sleep(1000);
                        }
                    }
                }
            } catch (Exception e) {
                logger.error("Something went wrong", e);
                failures.add(e);
            } finally {
                try {
                    waitForOutstandingRequests(TimeValue.timeValueSeconds(60), updateRequestsOutstanding, maxUpdateRequests, "Update");
                    waitForOutstandingRequests(TimeValue.timeValueSeconds(60), deleteRequestsOutstanding, maxDeleteRequests, "Delete");
                } catch (ElasticsearchTimeoutException ete) {
                    failures.add(ete);
                }
                latch.countDown();
            }
        }

        private void incrementMapValue(int j, Map<Integer, Integer> map) {
            if (!map.containsKey(j)) {
                map.put(j, 0);
            }
            map.put(j, map.get(j) + 1);
        }

        private void waitForOutstandingRequests(TimeValue timeOut, Semaphore requestsOutstanding, int maxRequests, String name) {
            long start = System.currentTimeMillis();
            do {
                long msRemaining = timeOut.getMillis() - (System.currentTimeMillis() - start);
                logger.info("[{}] going to try and acquire [{}] in [{}]ms [{}] available to acquire right now", name, maxRequests, msRemaining, requestsOutstanding.availablePermits());
                try {
                    requestsOutstanding.tryAcquire(maxRequests, msRemaining, TimeUnit.MILLISECONDS);
                    return;
                } catch (InterruptedException ie) {
                //Just keep swimming
                }
            } while ((System.currentTimeMillis() - start) < timeOut.getMillis());
            throw new ElasticsearchTimeoutException("Requests were still outstanding after the timeout [" + timeOut + "] for type [" + name + "]");
        }
    }
    final List<UpdateThread> threads = new ArrayList<>();
    for (int i = 0; i < numberOfThreads; i++) {
        UpdateThread ut = new UpdateThread(numberOfIdsPerThread, numberOfUpdatesPerId);
        ut.start();
        threads.add(ut);
    }
    startLatch.countDown();
    latch.await();
    for (UpdateThread ut : threads) {
        //Threads should have finished because of the latch.await
        ut.join();
    }
    //aquiring the request outstanding semaphores.
    for (Throwable throwable : failures) {
        logger.info("Captured failure on concurrent update:", throwable);
    }
    assertThat(failures.size(), equalTo(0));
    //All the previous operations should be complete or failed at this point
    for (int i = 0; i < numberOfIdsPerThread; ++i) {
        UpdateResponse ur = client().prepareUpdate("test", "type1", Integer.toString(i)).setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap())).setRetryOnConflict(Integer.MAX_VALUE).setUpsert(jsonBuilder().startObject().field("field", 1).endObject()).execute().actionGet();
    }
    refresh();
    for (int i = 0; i < numberOfIdsPerThread; ++i) {
        int totalFailures = 0;
        GetResponse response = client().prepareGet("test", "type1", Integer.toString(i)).execute().actionGet();
        if (response.isExists()) {
            assertThat(response.getId(), equalTo(Integer.toString(i)));
            int expectedVersion = (numberOfThreads * numberOfUpdatesPerId * 2) + 1;
            for (UpdateThread ut : threads) {
                if (ut.failedMap.containsKey(i)) {
                    totalFailures += ut.failedMap.get(i);
                }
            }
            expectedVersion -= totalFailures;
            logger.error("Actual version [{}] Expected version [{}] Total failures [{}]", response.getVersion(), expectedVersion, totalFailures);
            assertThat(response.getVersion(), equalTo((long) expectedVersion));
            assertThat(response.getVersion() + totalFailures, equalTo((long) ((numberOfUpdatesPerId * numberOfThreads * 2) + 1)));
        }
    }
}

Example 62 with GetResponse

use of org.elasticsearch.action.get.GetResponse in project elasticsearch by elastic.

the class UpdateIT method testVersionedUpdate.

public void testVersionedUpdate() throws Exception {
    assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
    ensureGreen();
    // version is now 1
    index("test", "type", "1", "text", "value");
    assertThrows(client().prepareUpdate(indexOrAlias(), "type", "1").setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v2"))).setVersion(2).execute(), VersionConflictEngineException.class);
    client().prepareUpdate(indexOrAlias(), "type", "1").setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v2"))).setVersion(1).get();
    assertThat(client().prepareGet("test", "type", "1").get().getVersion(), equalTo(2L));
    // and again with a higher version..
    client().prepareUpdate(indexOrAlias(), "type", "1").setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v3"))).setVersion(2).get();
    assertThat(client().prepareGet("test", "type", "1").get().getVersion(), equalTo(3L));
    // after delete
    client().prepareDelete("test", "type", "1").get();
    assertThrows(client().prepareUpdate("test", "type", "1").setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v2"))).setVersion(3).execute(), DocumentMissingException.class);
    // external versioning
    client().prepareIndex("test", "type", "2").setSource("text", "value").setVersion(10).setVersionType(VersionType.EXTERNAL).get();
    assertThrows(client().prepareUpdate(indexOrAlias(), "type", "2").setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v2"))).setVersion(2).setVersionType(VersionType.EXTERNAL).execute(), ActionRequestValidationException.class);
    GetResponse get = get("test", "type", "2");
    assertThat(get.getVersion(), equalTo(10L));
    assertThat((String) get.getSource().get("text"), equalTo("value"));
    // upserts - the combination with versions is a bit weird. Test are here to ensure we do not change our behavior unintentionally
    // With internal versions, tt means "if object is there with version X, update it or explode. If it is not there, index.
    client().prepareUpdate(indexOrAlias(), "type", "3").setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v2"))).setVersion(10).setUpsert("{ \"text\": \"v0\" }", XContentType.JSON).get();
    get = get("test", "type", "3");
    assertThat(get.getVersion(), equalTo(1L));
    assertThat((String) get.getSource().get("text"), equalTo("v0"));
    // retry on conflict is rejected:
    assertThrows(client().prepareUpdate(indexOrAlias(), "type", "1").setVersion(10).setRetryOnConflict(5), ActionRequestValidationException.class);
}

Example 63 with GetResponse

use of org.elasticsearch.action.get.GetResponse in project elasticsearch by elastic.

the class UpdateIT method testUpdate.

public void testUpdate() throws Exception {
    createTestIndex();
    ensureGreen();
    try {
        client().prepareUpdate(indexOrAlias(), "type1", "1").setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap())).execute().actionGet();
        fail();
    } catch (DocumentMissingException e) {
    // all is well
    }
    client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();
    UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap())).execute().actionGet();
    assertThat(updateResponse.getVersion(), equalTo(2L));
    assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
    assertThat(updateResponse.getIndex(), equalTo("test"));
    for (int i = 0; i < 5; i++) {
        GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
        assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("2"));
    }
    Map<String, Object> params = new HashMap<>();
    params.put("inc", 3);
    updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setScript(new Script(ScriptType.INLINE, "field_inc", "field", params)).execute().actionGet();
    assertThat(updateResponse.getVersion(), equalTo(3L));
    assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
    assertThat(updateResponse.getIndex(), equalTo("test"));
    for (int i = 0; i < 5; i++) {
        GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
        assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("5"));
    }
    // check noop
    updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("_ctx", Collections.singletonMap("op", "none")))).execute().actionGet();
    assertThat(updateResponse.getVersion(), equalTo(3L));
    assertEquals(DocWriteResponse.Result.NOOP, updateResponse.getResult());
    assertThat(updateResponse.getIndex(), equalTo("test"));
    for (int i = 0; i < 5; i++) {
        GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
        assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("5"));
    }
    // check delete
    updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("_ctx", Collections.singletonMap("op", "delete")))).execute().actionGet();
    assertThat(updateResponse.getVersion(), equalTo(4L));
    assertEquals(DocWriteResponse.Result.DELETED, updateResponse.getResult());
    assertThat(updateResponse.getIndex(), equalTo("test"));
    for (int i = 0; i < 5; i++) {
        GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
        assertThat(getResponse.isExists(), equalTo(false));
    }
    // check fields parameter
    client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();
    updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap())).setFields("field").setFetchSource(true).execute().actionGet();
    assertThat(updateResponse.getIndex(), equalTo("test"));
    assertThat(updateResponse.getGetResult(), notNullValue());
    assertThat(updateResponse.getGetResult().getIndex(), equalTo("test"));
    assertThat(updateResponse.getGetResult().sourceRef(), notNullValue());
    assertThat(updateResponse.getGetResult().field("field").getValue(), notNullValue());
    // check _source parameter
    client().prepareIndex("test", "type1", "1").setSource("field1", 1, "field2", 2).execute().actionGet();
    updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setScript(new Script(ScriptType.INLINE, "field_inc", "field1", Collections.emptyMap())).setFetchSource("field1", "field2").get();
    assertThat(updateResponse.getIndex(), equalTo("test"));
    assertThat(updateResponse.getGetResult(), notNullValue());
    assertThat(updateResponse.getGetResult().getIndex(), equalTo("test"));
    assertThat(updateResponse.getGetResult().sourceRef(), notNullValue());
    assertThat(updateResponse.getGetResult().field("field1"), nullValue());
    assertThat(updateResponse.getGetResult().sourceAsMap().size(), equalTo(1));
    assertThat(updateResponse.getGetResult().sourceAsMap().get("field1"), equalTo(2));
    // check updates without script
    // add new field
    client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();
    updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("field2", 2).endObject()).execute().actionGet();
    for (int i = 0; i < 5; i++) {
        GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
        assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("1"));
        assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2"));
    }
    // change existing field
    updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("field", 3).endObject()).execute().actionGet();
    for (int i = 0; i < 5; i++) {
        GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
        assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("3"));
        assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2"));
    }
    // recursive map
    Map<String, Object> testMap = new HashMap<>();
    Map<String, Object> testMap2 = new HashMap<>();
    Map<String, Object> testMap3 = new HashMap<>();
    testMap3.put("commonkey", testMap);
    testMap3.put("map3", 5);
    testMap2.put("map2", 6);
    testMap.put("commonkey", testMap2);
    testMap.put("map1", 8);
    client().prepareIndex("test", "type1", "1").setSource("map", testMap).execute().actionGet();
    updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("map", testMap3).endObject()).execute().actionGet();
    for (int i = 0; i < 5; i++) {
        GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
        Map map1 = (Map) getResponse.getSourceAsMap().get("map");
        assertThat(map1.size(), equalTo(3));
        assertThat(map1.containsKey("map1"), equalTo(true));
        assertThat(map1.containsKey("map3"), equalTo(true));
        assertThat(map1.containsKey("commonkey"), equalTo(true));
        Map map2 = (Map) map1.get("commonkey");
        assertThat(map2.size(), equalTo(3));
        assertThat(map2.containsKey("map1"), equalTo(true));
        assertThat(map2.containsKey("map2"), equalTo(true));
        assertThat(map2.containsKey("commonkey"), equalTo(true));
    }
}

Example 64 with GetResponse

use of org.elasticsearch.action.get.GetResponse in project elasticsearch by elastic.

the class UpdateIT method testScriptedUpsert.

public void testScriptedUpsert() throws Exception {
    createTestIndex();
    ensureGreen();
    // Script logic is
    // 1) New accounts take balance from "balance" in upsert doc and first payment is charged at 50%
    // 2) Existing accounts subtract full payment from balance stored in elasticsearch
    int openingBalance = 10;
    Map<String, Object> params = new HashMap<>();
    params.put("payment", 2);
    // Pay money from what will be a new account and opening balance comes from upsert doc
    // provided by client
    UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setUpsert(XContentFactory.jsonBuilder().startObject().field("balance", openingBalance).endObject()).setScriptedUpsert(true).setScript(new Script(ScriptType.INLINE, "scripted_upsert", "", params)).execute().actionGet();
    assertEquals(DocWriteResponse.Result.CREATED, updateResponse.getResult());
    assertThat(updateResponse.getIndex(), equalTo("test"));
    for (int i = 0; i < 5; i++) {
        GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
        assertThat(getResponse.getSourceAsMap().get("balance").toString(), equalTo("9"));
    }
    // Now pay money for an existing account where balance is stored in es
    updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setUpsert(XContentFactory.jsonBuilder().startObject().field("balance", openingBalance).endObject()).setScriptedUpsert(true).setScript(new Script(ScriptType.INLINE, "scripted_upsert", "", params)).execute().actionGet();
    assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
    assertThat(updateResponse.getIndex(), equalTo("test"));
    for (int i = 0; i < 5; i++) {
        GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
        assertThat(getResponse.getSourceAsMap().get("balance").toString(), equalTo("7"));
    }
}

Example 65 with GetResponse

use of org.elasticsearch.action.get.GetResponse in project elasticsearch by elastic.

the class UpdateIT method testUpsert.

public void testUpsert() throws Exception {
    createTestIndex();
    ensureGreen();
    UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setUpsert(XContentFactory.jsonBuilder().startObject().field("field", 1).endObject()).setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap())).execute().actionGet();
    assertEquals(DocWriteResponse.Result.CREATED, updateResponse.getResult());
    assertThat(updateResponse.getIndex(), equalTo("test"));
    for (int i = 0; i < 5; i++) {
        GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
        assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("1"));
    }
    updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setUpsert(XContentFactory.jsonBuilder().startObject().field("field", 1).endObject()).setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap())).execute().actionGet();
    assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
    assertThat(updateResponse.getIndex(), equalTo("test"));
    for (int i = 0; i < 5; i++) {
        GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
        assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("2"));
    }
}