Examples with Response org.elasticsearch.client.Response used on opensource projects

Example 71 with Response

use of org.elasticsearch.client.Response in project pyramid by cheng-li.

the class Visualizer method getPositions.

private JsonArray getPositions(String esIndex, String id, String field, JsonElement keywords, int slop, boolean inOrder) {
    // TODO: Debug
    // System.out.println("ID: " + id);
    // System.out.println("Words: " + keywords);
    // System.out.println("slope: " + slop);
    // System.out.println("IN");
    JsonArray clauses = new JsonArray();
    String[] keywordsArr = keywords.getAsString().split("\\s");
    String queryJson = null;
    if (keywordsArr.length == 1) {
        queryJson = createUnigramQueryJson(id, field, keywordsArr[0]);
    } else {
        for (String keyword : keywordsArr) {
            JsonObject field_j = new JsonObject();
            field_j.add(field, gson.toJsonTree(keyword));
            JsonObject clause = new JsonObject();
            clause.add("span_term", field_j);
            clauses.add(clause);
        }
        // added to handle lucene changes that require a min. of 2 span fields
        // System.out.println("clauses_original: " + clauses);
        // if (clauses.size() == 1)
        // clauses.add(clauses.get(0));
        // System.out.println("clauses_new: " + clauses);
        queryJson = createQueryJson(id, inOrder, slop, field, clauses);
    }
    // System.out.println(queryJson);
    // try {
    // System.in.read();
    // } catch (IOException e) {
    // e.printStackTrace();
    // }
    HttpEntity httpEntity = new NStringEntity(queryJson, ContentType.APPLICATION_JSON);
    String responseStr = null;
    try {
        Response response = esClient.performRequest("GET", esIndex + "/" + Properties.DOCUMENT_TYPE + "/" + "_search", Collections.emptyMap(), httpEntity);
        responseStr = EntityUtils.toString(response.getEntity());
    } catch (IOException e) {
        e.printStackTrace();
    }
    JsonObject responseObj = jsonParser.parse(responseStr).getAsJsonObject();
    JsonArray hits = responseObj.get("hits").getAsJsonObject().get("hits").getAsJsonArray();
    if (hits.size() == 0) {
        return new JsonArray();
    }
    // positions is an array of arryas. not sure if inefficient. TODO: change later if inefficient
    JsonArray positions = new JsonArray();
    JsonObject hit0 = hits.get(0).getAsJsonObject();
    Set<String> fields = hit0.get("highlight").getAsJsonObject().keySet();
    for (String higlightField : fields) {
        String text = hit0.get("_source").getAsJsonObject().get(higlightField).getAsString();
        JsonArray highlights = hit0.get("highlight").getAsJsonObject().get(higlightField).getAsJsonArray();
        for (JsonElement hl_je : highlights) {
            String hl = hl_je.getAsString();
            // System.out.println("hl" + hl);
            // remove <em> and </em>
            String cleanHl = hl.replaceAll("<\\/?em>", "");
            int baseIndex = text.indexOf(cleanHl);
            if (baseIndex == -1)
                continue;
            while (hl.indexOf("<em>") != -1) {
                int start = hl.indexOf("<em>") + baseIndex;
                hl = hl.replaceFirst("<em>", "");
                int end = hl.indexOf("</em>") + baseIndex;
                hl = hl.replaceFirst("</em>", "");
                JsonArray curPos = gson.toJsonTree(new int[] { start, end }).getAsJsonArray();
                // System.out.println("curPos:" + curPos);
                // try {
                // System.in.read();
                // } catch (IOException e) {
                // e.printStackTrace();
                // }
                // positions is a array of arrays. (each curPos is an array)
                positions.add(curPos);
            }
        }
    }
    return positions;
}

Example 72 with Response

use of org.elasticsearch.client.Response in project pyramid by cheng-li.

the class Visualizer method createRow.

/*   corresponds to each iteration of the loop in createTable() of viusualizer.py   */
private JsonObject createRow(JsonObject jsonRow, String esIndex, String fields, int lineCount) {
    JsonObject thisRow = new JsonObject();
    Map<String, JsonElement> labelsMap = new HashMap<>();
    labelsMap.put("id", jsonRow.get("id"));
    labelsMap.put("internalId", jsonRow.get("internalId"));
    labelsMap.put("internalLabels", jsonRow.get("internalLabels"));
    JsonParser jp = new JsonParser();
    labelsMap.put("feedbackSelect", jp.parse("none"));
    // WARNING: "" is parsed to null in json
    labelsMap.put("feedbackText", jp.parse(""));
    /*  Internal Labels:  */
    Map<String, JsonElement> internalLabels = new HashMap<>();
    JsonArray labels_jsonArr = jsonRow.get("labels").getAsJsonArray();
    JsonArray internalLabels_jsonArr = jsonRow.get("internalLabels").getAsJsonArray();
    int totalLabels = labels_jsonArr.size();
    labelsMap.put("internalLabels", new JsonArray());
    for (int i = 0; i < totalLabels; i++) {
        internalLabels.put(internalLabels_jsonArr.get(i).getAsString(), labels_jsonArr.get(i));
        JsonObject jo = new JsonObject();
        jo.add(internalLabels_jsonArr.get(i).getAsString(), labels_jsonArr.get(i));
        labelsMap.get("internalLabels").getAsJsonArray().add(jo);
    }
    /*  Predictions:  */
    Map<String, JsonElement> predictions = new HashMap<>();
    JsonArray predictions_jsons = jsonRow.get("prediction").getAsJsonArray();
    JsonArray internalPredictions_jsons = jsonRow.get("internalPrediction").getAsJsonArray();
    int totalPredictions = predictions_jsons.size();
    labelsMap.put("predictions", new JsonArray());
    for (int i = 0; i < totalPredictions; i++) {
        // TODO: use getAsString() instead of toString() ?
        predictions.put(internalPredictions_jsons.get(i).getAsString(), predictions_jsons.get(i));
        JsonObject jo = new JsonObject();
        jo.add(internalPredictions_jsons.get(i).getAsString(), predictions_jsons.get(i));
        JsonArray ja = labelsMap.get("predictions").getAsJsonArray();
        ja.add(jo);
    }
    /*  overlap, recall and precision  */
    Set<String> internalLabelIds = new HashSet<>(internalLabels.keySet());
    Set<String> predictionsIds = new HashSet<>(predictions.keySet());
    Set<String> intersection = new HashSet<>(internalLabelIds);
    intersection.retainAll(predictionsIds);
    Set<String> union = new HashSet<>(internalLabelIds);
    union.addAll(predictionsIds);
    if (union.size() == 0)
        labelsMap.put("overlap", gson.toJsonTree("N/A"));
    else
        labelsMap.put("overlap", gson.toJsonTree(String.format("%.2f", (double) intersection.size() / union.size())));
    if (internalLabelIds.size() == 0)
        labelsMap.put("recall", gson.toJsonTree("N/A"));
    else
        labelsMap.put("recall", gson.toJsonTree(String.format("%.2f", (double) intersection.size() / internalLabelIds.size())));
    if (predictionsIds.size() == 0)
        labelsMap.put("precision", gson.toJsonTree("N/A"));
    else
        labelsMap.put("precision", gson.toJsonTree(String.format("%.2f", (double) intersection.size() / predictionsIds.size())));
    /*  probForPredictedLabels  */
    thisRow.add("probForPredictedLabels", jsonRow.get("probForPredictedLabels"));
    /*  column 2; predicted ranking  */
    JsonArray predictedRankings_ja = jsonRow.get("predictedRanking").getAsJsonArray();
    PredictedRanking[] predictedRankings = gson.fromJson(predictedRankings_ja, PredictedRanking[].class);
    PredictedRanking[] updatedPredictedRankings = new PredictedRanking[predictedRankings.length];
    List<Integer> r = new ArrayList<>(predictedRankings.length);
    JsonElement predictedRanking_json = null;
    for (int i = 0; i < predictedRankings.length; i++) {
        // ~ label in visualizer.py
        PredictedRanking predictedRanking = predictedRankings[i];
        predictedRanking_json = gson.toJsonTree(predictedRanking);
        JsonElement classIndex_je = predictedRanking_json.getAsJsonObject().get("classIndex");
        if (internalLabels_jsonArr.contains(classIndex_je) && internalPredictions_jsons.contains(classIndex_je)) {
            predictedRanking_json.getAsJsonObject().addProperty("type", "TP");
        } else if (!internalLabels_jsonArr.contains(classIndex_je) && internalPredictions_jsons.contains(classIndex_je))
            predictedRanking_json.getAsJsonObject().addProperty("type", "FP");
        else if (internalLabels_jsonArr.contains(classIndex_je) && !internalPredictions_jsons.contains(classIndex_je))
            predictedRanking_json.getAsJsonObject().addProperty("type", "FN");
        else
            predictedRanking_json.getAsJsonObject().addProperty("type", "");
        PredictedRanking newPredictedRanking = gson.fromJson(predictedRanking_json, PredictedRanking.class);
        updatedPredictedRankings[i] = newPredictedRanking;
        // updating r for some-more-labelsMap
        r.add(includesLabel(predictedRanking.className, internalLabels));
    }
    JsonArray updatedPredictedRankings_json = gson.toJsonTree(updatedPredictedRankings).getAsJsonArray();
    thisRow.add("predictedRanking", updatedPredictedRankings_json);
    /*   predictedLabelSetRankings   */
    JsonArray predictedLabelSetRankings = jsonRow.get("predictedLabelSetRanking").getAsJsonArray();
    // making a Set copy of these so that contains() is constant time:
    Set<Integer> internalLabels_ints = jsonArrToIntSet(internalLabels_jsonArr);
    Set<Integer> internalPredictions_ints = jsonArrToIntSet(internalPredictions_jsons);
    for (JsonElement labelsJElement : predictedLabelSetRankings) {
        JsonObject labels = labelsJElement.getAsJsonObject();
        // labels ~ labels in visualizer.py
        JsonArray predictedInternalLabels = labels.get("internalLabels").getAsJsonArray();
        labels.add("types", new JsonArray());
        for (JsonElement index_je : predictedInternalLabels) {
            int index = index_je.getAsInt();
            if (internalLabels_ints.contains(index) && internalPredictions_ints.contains(index))
                labels.get("types").getAsJsonArray().add("TP");
            else if (!internalLabels_ints.contains(index) && internalPredictions_ints.contains(index))
                labels.get("types").getAsJsonArray().add("FP");
            else if (internalLabels_ints.contains(index) && !internalPredictions_ints.contains(index))
                labels.get("types").getAsJsonArray().add("FN");
            else
                labels.get("types").getAsJsonArray().add("");
        }
    }
    thisRow.add("predictedLabelSetRanking", jsonRow.get("predictedLabelSetRanking"));
    /*   some-more-labelsMap   */
    double sumOfR = 0;
    double sumOfPrec = 0;
    double prec = 0;
    int last = 0;
    for (int i = 0; i < r.size(); i++) {
        if (r.get(i) == 1) {
            sumOfR += 1;
            prec = sumOfR / (i + 1);
            sumOfPrec += prec;
            last = i + 1;
        }
    }
    int intLblsSize = internalLabelIds.size();
    if (intLblsSize == 0)
        labelsMap.put("ap", gson.toJsonTree("N/A"));
    else
        labelsMap.put("ap", gson.toJsonTree(String.format("%.2f", (sumOfPrec / intLblsSize))));
    if (sumOfR < intLblsSize)
        labelsMap.put("rankoffullrecall", gson.toJsonTree("N/A"));
    else
        // warning: storing integer as string. might create problems later
        labelsMap.put("rankoffullrecall", gson.toJsonTree(last));
    thisRow.add("idlabels", gson.toJsonTree(labelsMap));
    /*   column 3 : ES   */
    Response response = null;
    String jsonResponse = null;
    try {
        response = esClient.performRequest("GET", esIndex + "/" + Properties.DOCUMENT_TYPE + "/" + URLEncoder.encode(labelsMap.get("id").getAsString(), "UTF-8"), Collections.emptyMap());
        jsonResponse = EntityUtils.toString(response.getEntity());
    } catch (IOException e) {
        e.printStackTrace();
    }
    JsonObject docSource = jsonParser.parse(jsonResponse).getAsJsonObject().get("_source").getAsJsonObject();
    thisRow.add("text", new JsonObject());
    thisRow.add("others", new JsonObject());
    for (Map.Entry<String, JsonElement> entry : docSource.entrySet()) {
        String k = entry.getKey();
        JsonElement v = entry.getValue();
        if (entry.getKey().equals(fields)) {
            thisRow.get("text").getAsJsonObject().add(k, v);
        } else
            thisRow.get("others").getAsJsonObject().add(k, v);
    }
    /*   column 4 - 7 TP FP FN TN   */
    createTFPNColumns(esIndex, jsonRow, lineCount, thisRow);
    return thisRow;
}

Example 73 with Response

use of org.elasticsearch.client.Response in project beam by apache.

the class ElasticsearchIOTestCommon method testDefaultRetryPredicate.

/**
 * Test that the default predicate correctly parses chosen error code.
 */
void testDefaultRetryPredicate(RestClient restClient) throws IOException {
    HttpEntity entity1 = new NStringEntity(BAD_REQUEST, ContentType.APPLICATION_JSON);
    Request request = new Request("POST", "/_bulk");
    request.addParameters(Collections.emptyMap());
    request.setEntity(entity1);
    Response response1 = restClient.performRequest(request);
    assertTrue(CUSTOM_RETRY_PREDICATE.test(response1.getEntity()));
    HttpEntity entity2 = new NStringEntity(OK_REQUEST, ContentType.APPLICATION_JSON);
    request.setEntity(entity2);
    Response response2 = restClient.performRequest(request);
    assertFalse(DEFAULT_RETRY_PREDICATE.test(response2.getEntity()));
}

Example 74 with Response

use of org.elasticsearch.client.Response in project beam by apache.

the class ElasticsearchIO method getBackendVersion.

static int getBackendVersion(ConnectionConfiguration connectionConfiguration) {
    try (RestClient restClient = connectionConfiguration.createClient()) {
        Request request = new Request("GET", "");
        Response response = restClient.performRequest(request);
        JsonNode jsonNode = parseResponse(response.getEntity());
        int backendVersion = Integer.parseInt(jsonNode.path("version").path("number").asText().substring(0, 1));
        checkArgument(VALID_CLUSTER_VERSIONS.contains(backendVersion), "The Elasticsearch version to connect to is %s.x. " + "This version of the ElasticsearchIO is only compatible with " + "Elasticsearch v7.x, v6.x, v5.x and v2.x", backendVersion);
        return backendVersion;
    } catch (IOException e) {
        throw new IllegalArgumentException("Cannot get Elasticsearch version", e);
    }
}

Example 75 with Response

use of org.elasticsearch.client.Response in project hazelcast by hazelcast.

the class ElasticCatClientTest method response.

private Response response(String json) throws IOException {
    Response response = mock(Response.class, RETURNS_DEEP_STUBS);
    when(response.getEntity().getContent()).thenReturn(new FileInputStream("src/test/resources/mock_es_responses/" + json));
    return response;
}