Examples with IndexRequestBuilder org.elasticsearch.action.index.IndexRequestBuilder used on opensource projects

Example 36 with IndexRequestBuilder

use of org.elasticsearch.action.index.IndexRequestBuilder in project elasticsearch by elastic.

the class DeleteByQueryConcurrentTests method testConcurrentDeleteByQueriesOnSameDocs.

public void testConcurrentDeleteByQueriesOnSameDocs() throws Throwable {
    final long docs = randomIntBetween(50, 100);
    List<IndexRequestBuilder> builders = new ArrayList<>();
    for (int i = 0; i < docs; i++) {
        builders.add(client().prepareIndex("test", "doc", String.valueOf(i)).setSource("foo", "bar"));
    }
    indexRandom(true, true, true, builders);
    final Thread[] threads = new Thread[scaledRandomIntBetween(2, 9)];
    final CountDownLatch start = new CountDownLatch(1);
    final MatchQueryBuilder query = matchQuery("foo", "bar");
    final AtomicLong deleted = new AtomicLong(0);
    for (int t = 0; t < threads.length; t++) {
        Runnable r = () -> {
            try {
                start.await();
                BulkByScrollResponse response = deleteByQuery().source("test").filter(query).refresh(true).get();
                // Some deletions might fail due to version conflict, but
                // what matters here is the total of successful deletions
                deleted.addAndGet(response.getDeleted());
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            }
        };
        threads[t] = new Thread(r);
        threads[t].start();
    }
    start.countDown();
    for (Thread thread : threads) {
        thread.join();
    }
    assertHitCount(client().prepareSearch("test").setSize(0).get(), 0L);
    assertThat(deleted.get(), equalTo(docs));
}

Example 37 with IndexRequestBuilder

use of org.elasticsearch.action.index.IndexRequestBuilder in project elasticsearch by elastic.

the class DeleteByQueryConcurrentTests method testConcurrentDeleteByQueriesOnDifferentDocs.

public void testConcurrentDeleteByQueriesOnDifferentDocs() throws Throwable {
    final Thread[] threads = new Thread[scaledRandomIntBetween(2, 5)];
    final long docs = randomIntBetween(1, 50);
    List<IndexRequestBuilder> builders = new ArrayList<>();
    for (int i = 0; i < docs; i++) {
        for (int t = 0; t < threads.length; t++) {
            builders.add(client().prepareIndex("test", "doc").setSource("field", t));
        }
    }
    indexRandom(true, true, true, builders);
    final CountDownLatch start = new CountDownLatch(1);
    for (int t = 0; t < threads.length; t++) {
        final int threadNum = t;
        assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.termQuery("field", threadNum)).get(), docs);
        Runnable r = () -> {
            try {
                start.await();
                assertThat(deleteByQuery().source("_all").filter(termQuery("field", threadNum)).refresh(true).get(), matcher().deleted(docs));
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            }
        };
        threads[t] = new Thread(r);
        threads[t].start();
    }
    start.countDown();
    for (Thread thread : threads) {
        thread.join();
    }
    for (int t = 0; t < threads.length; t++) {
        assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.termQuery("field", t)).get(), 0);
    }
}

Example 38 with IndexRequestBuilder

use of org.elasticsearch.action.index.IndexRequestBuilder in project elasticsearch by elastic.

the class ReindexBasicTests method testCopyManyWithSlices.

public void testCopyManyWithSlices() throws Exception {
    int workers = between(2, 10);
    List<IndexRequestBuilder> docs = new ArrayList<>();
    int max = between(150, 500);
    for (int i = 0; i < max; i++) {
        docs.add(client().prepareIndex("source", "test", Integer.toString(i)).setSource("foo", "a"));
    }
    indexRandom(true, docs);
    assertHitCount(client().prepareSearch("source").setSize(0).get(), max);
    // Copy all the docs
    ReindexRequestBuilder copy = reindex().source("source").destination("dest", "all").refresh(true).setSlices(workers);
    // Use a small batch size so we have to use more than one batch
    copy.source().setSize(5);
    assertThat(copy.get(), matcher().created(max).batches(greaterThanOrEqualTo(max / 5)).slices(hasSize(workers)));
    assertHitCount(client().prepareSearch("dest").setTypes("all").setSize(0).get(), max);
    // Copy some of the docs
    int half = max / 2;
    copy = reindex().source("source").destination("dest", "half").refresh(true).setSlices(workers);
    // Use a small batch size so we have to use more than one batch
    copy.source().setSize(5);
    // The real "size" of the request.
    copy.size(half);
    BulkByScrollResponse response = copy.get();
    assertThat(response, matcher().created(lessThanOrEqualTo((long) half)).slices(hasSize(workers)));
    assertHitCount(client().prepareSearch("dest").setTypes("half").setSize(0).get(), response.getCreated());
}

Example 39 with IndexRequestBuilder

use of org.elasticsearch.action.index.IndexRequestBuilder in project elasticsearch by elastic.

the class FieldStatsIntegrationIT method testRandom.

public void testRandom() throws Exception {
    assertAcked(prepareCreate("test").addMapping("test", "string", "type=text", "date", "type=date", "double", "type=double", "half_float", "type=half_float", "float", "type=float", "long", "type=long", "integer", "type=integer", "short", "type=short", "byte", "type=byte", "location", "type=geo_point"));
    ensureGreen("test");
    // index=false
    assertAcked(prepareCreate("test1").addMapping("test", "string", "type=text,index=false", "date", "type=date,index=false", "double", "type=double,index=false", "half_float", "type=half_float", "float", "type=float,index=false", "long", "type=long,index=false", "integer", "type=integer,index=false", "short", "type=short,index=false", "byte", "type=byte,index=false", "location", "type=geo_point,index=false"));
    ensureGreen("test1");
    // no value indexed
    assertAcked(prepareCreate("test3").addMapping("test", "string", "type=text,index=false", "date", "type=date,index=false", "double", "type=double,index=false", "half_float", "type=half_float", "float", "type=float,index=false", "long", "type=long,index=false", "integer", "type=integer,index=false", "short", "type=short,index=false", "byte", "type=byte,index=false", "location", "type=geo_point,index=false"));
    ensureGreen("test3");
    long minByte = Byte.MAX_VALUE;
    long maxByte = Byte.MIN_VALUE;
    long minShort = Short.MAX_VALUE;
    long maxShort = Short.MIN_VALUE;
    long minInt = Integer.MAX_VALUE;
    long maxInt = Integer.MIN_VALUE;
    long minLong = Long.MAX_VALUE;
    long maxLong = Long.MIN_VALUE;
    double minHalfFloat = Double.POSITIVE_INFINITY;
    double maxHalfFloat = Double.NEGATIVE_INFINITY;
    double minFloat = Double.POSITIVE_INFINITY;
    double maxFloat = Double.NEGATIVE_INFINITY;
    double minDouble = Double.POSITIVE_INFINITY;
    double maxDouble = Double.NEGATIVE_INFINITY;
    GeoPoint minLoc = new GeoPoint(90, 180);
    GeoPoint maxLoc = new GeoPoint(-90, -180);
    String minString = new String(Character.toChars(1114111));
    String maxString = "0";
    int numDocs = scaledRandomIntBetween(128, 1024);
    List<IndexRequestBuilder> request = new ArrayList<>(numDocs);
    for (int doc = 0; doc < numDocs; doc++) {
        byte b = randomByte();
        minByte = Math.min(minByte, b);
        maxByte = Math.max(maxByte, b);
        short s = randomShort();
        minShort = Math.min(minShort, s);
        maxShort = Math.max(maxShort, s);
        int i = randomInt();
        minInt = Math.min(minInt, i);
        maxInt = Math.max(maxInt, i);
        long l = randomLong();
        minLong = Math.min(minLong, l);
        maxLong = Math.max(maxLong, l);
        float hf = randomFloat();
        hf = HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(hf));
        minHalfFloat = Math.min(minHalfFloat, hf);
        maxHalfFloat = Math.max(maxHalfFloat, hf);
        float f = randomFloat();
        minFloat = Math.min(minFloat, f);
        maxFloat = Math.max(maxFloat, f);
        double d = randomDouble();
        minDouble = Math.min(minDouble, d);
        maxDouble = Math.max(maxDouble, d);
        GeoPoint loc = RandomGeoGenerator.randomPoint(random());
        minLoc.reset(Math.min(loc.lat(), minLoc.lat()), Math.min(loc.lon(), minLoc.lon()));
        maxLoc.reset(Math.max(loc.lat(), maxLoc.lat()), Math.max(loc.lon(), maxLoc.lon()));
        String str = randomRealisticUnicodeOfLength(3);
        if (str.compareTo(minString) < 0) {
            minString = str;
        }
        if (str.compareTo(maxString) > 0) {
            maxString = str;
        }
        request.add(client().prepareIndex("test", "test", Integer.toString(doc)).setSource("byte", b, "short", s, "integer", i, "long", l, "half_float", hf, "float", f, "double", d, "location", loc, "string", str));
    }
    indexRandom(true, false, request);
    FieldStatsResponse response = client().prepareFieldStats().setFields("byte", "short", "integer", "long", "half_float", "float", "double", "location", "string").get();
    assertAllSuccessful(response);
    for (FieldStats<?> stats : response.getAllFieldStats().values()) {
        assertThat(stats.getMaxDoc(), equalTo((long) numDocs));
        assertThat(stats.getDocCount(), equalTo((long) numDocs));
        assertThat(stats.getDensity(), equalTo(100));
    }
    assertThat(response.getAllFieldStats().get("byte").getMinValue(), equalTo(minByte));
    assertThat(response.getAllFieldStats().get("byte").getMaxValue(), equalTo(maxByte));
    assertThat(response.getAllFieldStats().get("byte").getDisplayType(), equalTo("integer"));
    assertThat(response.getAllFieldStats().get("short").getMinValue(), equalTo(minShort));
    assertThat(response.getAllFieldStats().get("short").getMaxValue(), equalTo(maxShort));
    assertThat(response.getAllFieldStats().get("short").getDisplayType(), equalTo("integer"));
    assertThat(response.getAllFieldStats().get("integer").getMinValue(), equalTo(minInt));
    assertThat(response.getAllFieldStats().get("integer").getMaxValue(), equalTo(maxInt));
    assertThat(response.getAllFieldStats().get("integer").getDisplayType(), equalTo("integer"));
    assertThat(response.getAllFieldStats().get("long").getMinValue(), equalTo(minLong));
    assertThat(response.getAllFieldStats().get("long").getMaxValue(), equalTo(maxLong));
    assertThat(response.getAllFieldStats().get("long").getDisplayType(), equalTo("integer"));
    assertThat(response.getAllFieldStats().get("half_float").getMinValue(), equalTo(minHalfFloat));
    assertThat(response.getAllFieldStats().get("half_float").getMaxValue(), equalTo(maxHalfFloat));
    assertThat(response.getAllFieldStats().get("half_float").getDisplayType(), equalTo("float"));
    assertThat(response.getAllFieldStats().get("float").getMinValue(), equalTo(minFloat));
    assertThat(response.getAllFieldStats().get("float").getMaxValue(), equalTo(maxFloat));
    assertThat(response.getAllFieldStats().get("float").getDisplayType(), equalTo("float"));
    assertThat(response.getAllFieldStats().get("double").getMinValue(), equalTo(minDouble));
    assertThat(response.getAllFieldStats().get("double").getMaxValue(), equalTo(maxDouble));
    assertThat(response.getAllFieldStats().get("double").getDisplayType(), equalTo("float"));
    assertThat(((GeoPoint) response.getAllFieldStats().get("location").getMinValue()).lat(), closeTo(minLoc.lat(), 1E-5));
    assertThat(((GeoPoint) response.getAllFieldStats().get("location").getMinValue()).lon(), closeTo(minLoc.lon(), 1E-5));
    assertThat(((GeoPoint) response.getAllFieldStats().get("location").getMaxValue()).lat(), closeTo(maxLoc.lat(), 1E-5));
    assertThat(((GeoPoint) response.getAllFieldStats().get("location").getMaxValue()).lon(), closeTo(maxLoc.lon(), 1E-5));
    assertThat(response.getAllFieldStats().get("location").getDisplayType(), equalTo("geo_point"));
}

Example 40 with IndexRequestBuilder

use of org.elasticsearch.action.index.IndexRequestBuilder in project elasticsearch by elastic.

the class IndexWithShadowReplicasIT method testDeletingClosedIndexRemovesFiles.

public void testDeletingClosedIndexRemovesFiles() throws Exception {
    Path dataPath = createTempDir();
    Settings nodeSettings = nodeSettings(dataPath.getParent());
    final int numNodes = randomIntBetween(2, 5);
    logger.info("--> starting {} nodes", numNodes);
    final List<String> nodes = internalCluster().startNodes(numNodes, nodeSettings);
    final String IDX = "test";
    final Tuple<Integer, Integer> numPrimariesAndReplicas = randomPrimariesAndReplicas(numNodes);
    final int numPrimaries = numPrimariesAndReplicas.v1();
    final int numReplicas = numPrimariesAndReplicas.v2();
    logger.info("--> creating index {} with {} primary shards and {} replicas", IDX, numPrimaries, numReplicas);
    assert numPrimaries > 0;
    assert numReplicas >= 0;
    Settings idxSettings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numPrimaries).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numReplicas).put(IndexMetaData.SETTING_DATA_PATH, dataPath.toAbsolutePath().toString()).put(IndexMetaData.SETTING_SHADOW_REPLICAS, true).put(IndexMetaData.SETTING_SHARED_FILESYSTEM, true).build();
    prepareCreate(IDX).setSettings(idxSettings).addMapping("doc", "foo", "type=text").get();
    ensureGreen(IDX);
    int docCount = randomIntBetween(10, 100);
    List<IndexRequestBuilder> builders = new ArrayList<>();
    for (int i = 0; i < docCount; i++) {
        builders.add(client().prepareIndex(IDX, "doc", i + "").setSource("foo", "bar"));
    }
    indexRandom(true, true, true, builders);
    flushAndRefresh(IDX);
    logger.info("--> closing index {}", IDX);
    client().admin().indices().prepareClose(IDX).get();
    ensureGreen(IDX);
    logger.info("--> deleting closed index");
    client().admin().indices().prepareDelete(IDX).get();
    assertAllIndicesRemovedAndDeletionCompleted(internalCluster().getInstances(IndicesService.class));
    assertPathHasBeenCleared(dataPath);
    assertIndicesDirsDeleted(nodes);
}