Examples with Tag com.srotya.sidewinder.core.rpc.Tag used on opensource projects

Example 11 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class QAInMemoryByzantineDefaults method testSingleSeriesWritesQueryGrafana.

@SuppressWarnings("unchecked")
@Test
public void testSingleSeriesWritesQueryGrafana() throws KeyManagementException, ClientProtocolException, NoSuchAlgorithmException, KeyStoreException, MalformedURLException, IOException, ParseException, InterruptedException {
    long sts = 1497720452566L;
    SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
    format.setTimeZone(TimeZone.getTimeZone("utc"));
    HttpPost post = new HttpPost("http://localhost:" + PORT + "/influx?db=qaSingleSeries");
    CloseableHttpResponse response = TestUtils.makeRequest(post);
    assertEquals(400, response.getStatusLine().getStatusCode());
    post = new HttpPost("http://localhost:" + PORT + "/influx?db=qaSingleSeries");
    post.setEntity(new StringEntity("cpu,host=server01,region=uswest value=1i 1497720452566000000\n" + "cpu,host=server02,region=uswest value=1i 1497720452566000000\n" + "cpu,host=server03,region=uswest value=1i 1497720452566000000\n" + "cpu,host=server01,region=uswest value=1i 1497720453566000000\n" + "cpu,host=server02,region=uswest value=1i 1497720453566000000\n" + "cpu,host=server03,region=uswest value=1i 1497720453566000000"));
    response = TestUtils.makeRequest(post);
    assertEquals(204, response.getStatusLine().getStatusCode());
    post = new HttpPost("http://localhost:" + PORT + "/qaSingleSeries/query/measurements");
    post.setHeader("Content-Type", "application/json");
    response = TestUtils.makeRequest(post);
    assertEquals(200, response.getStatusLine().getStatusCode());
    Gson gson = new Gson();
    JsonArray ary = gson.fromJson(EntityUtils.toString(response.getEntity()), JsonArray.class);
    assertEquals("cpu", ary.get(0).getAsString());
    post = new HttpPost("http://localhost:" + PORT + "/qaSingleSeries/query/tags");
    post.setHeader("Content-Type", "application/json");
    post.setEntity(new StringEntity("{ \"target\":\"cpu\" }"));
    response = TestUtils.makeRequest(post);
    assertEquals(200, response.getStatusLine().getStatusCode());
    ary = gson.fromJson(EntityUtils.toString(response.getEntity()), JsonArray.class);
    assertEquals(2, ary.size());
    String payload = "{\"panelId\":2,\"range\":{\"from\":\"%s\",\"to\":\"%s\",\"raw\":{\"from\":\"now-5m\",\"to\":\"now\"}},\"rangeRaw\":{\"from\":\"now-5m\",\"to\":\"now\"},\"interval\":\"200ms\",\"intervalMs\":200,\"targets\":[{\"target\":\"cpu\",\"filters\":[],\"aggregator\":{\"args\":[{\"index\":0,\"type\":\"int\",\"value\":\"20\"}],\"name\":\"none\",\"unit\":\"secs\"},\"field\":\"value\",\"refId\":\"A\",\"type\":\"timeserie\"}],\"format\":\"json\",\"maxDataPoints\":1280}";
    post = new HttpPost("http://localhost:" + PORT + "/qaSingleSeries/query");
    post.setHeader("Content-Type", "application/json");
    post.setEntity(new StringEntity(String.format(payload, format.format(new Date(sts - 60_000)), format.format(new Date(sts + 60_000)))));
    response = TestUtils.makeRequest(post);
    assertEquals(200, response.getStatusLine().getStatusCode());
    ary = gson.fromJson(EntityUtils.toString(response.getEntity()), JsonArray.class);
    assertEquals(3, ary.size());
    int i = 0;
    for (JsonElement ele : ary) {
        i += ele.getAsJsonObject().get("datapoints").getAsJsonArray().size();
    }
    assertEquals(6, i);
    response = TestUtils.makeRequest(new HttpGet("http://localhost:" + PORT + "/databases/qaSingleSeries/measurements/cpu/fields/value?startTime=" + (sts - 2000) + "&endTime=" + (sts + 2000)));
    ary = gson.fromJson(EntityUtils.toString(response.getEntity()), JsonArray.class);
    Set<Tag> tag = new HashSet<>(Arrays.asList(Tag.newBuilder().setTagKey("host").setTagValue("server02").build(), Tag.newBuilder().setTagKey("host").setTagValue("server01").build(), Tag.newBuilder().setTagKey("host").setTagValue("server03").build(), Tag.newBuilder().setTagKey("region").setTagValue("uswest").build()));
    Iterator<JsonElement> itr = ary.iterator();
    i = 0;
    while (itr.hasNext()) {
        JsonObject obj = itr.next().getAsJsonObject();
        assertEquals("cpu", obj.get("measurementName").getAsString());
        assertEquals("value", obj.get("valueFieldName").getAsString());
        ary = obj.get("tags").getAsJsonArray();
        for (JsonElement ele : ary) {
            Tag tagObj = gson.fromJson(ele, Tag.class);
            assertTrue(ele + " ", tag.contains(tagObj));
        }
        i += obj.get("dataPoints").getAsJsonArray().size();
        JsonArray ary2 = obj.get("dataPoints").getAsJsonArray();
        for (int j = 0; j < ary2.size(); j++) {
            JsonElement ele = ary2.get(j);
            assertEquals(1, ele.getAsJsonObject().get("value").getAsInt());
            long ts = 1497720452566L + j * 1000;
            assertEquals(ts, ele.getAsJsonObject().get("timestamp").getAsLong());
        }
    }
    assertEquals(6, i);
    HttpGet get = new HttpGet("http://localhost:" + PORT + "/qaSingleSeries/hc");
    get.setHeader("Content-Type", "application/json");
    response = TestUtils.makeRequest(get);
    assertEquals(200, response.getStatusLine().getStatusCode());
    payload = "{\"panelId\":1,\"range\":{\"from\":\"%s\",\"to\":\"%s\"," + "\"raw\":{\"from\":\"now-6h\",\"to\":\"now\"}}," + "\"rangeRaw\":{\"from\":\"now-6h\",\"to\":\"now\"}," + "\"interval\":\"15s\",\"intervalMs\":15000,\"targets\":[{\"refId\":\"A\",\"raw\":\"cpu.value\",\"rawQuery\":true,\"type\":\"timeserie\"}]," + "\"format\":\"json\",\"maxDataPoints\":1272}";
    post = new HttpPost("http://localhost:" + PORT + "/qaSingleSeries/query");
    post.setHeader("Content-Type", "application/json");
    post.setEntity(new StringEntity(String.format(payload, format.format(new Date(sts - 60_000)), format.format(new Date(sts + 60_000)))));
    response = TestUtils.makeRequest(post);
    assertEquals(200, response.getStatusLine().getStatusCode());
    ary = gson.fromJson(EntityUtils.toString(response.getEntity()), JsonArray.class);
    assertEquals(3, ary.size());
    i = 0;
    for (JsonElement ele : ary) {
        i += ele.getAsJsonObject().get("datapoints").getAsJsonArray().size();
    }
    assertEquals(6, i);
    post = new HttpPost("http://localhost:" + PORT + "/qaSingleSeries/query/measurements");
    post.setHeader("Content-Type", "application/json");
    response = TestUtils.makeRequest(post);
    assertEquals(200, response.getStatusLine().getStatusCode());
    post = new HttpPost("http://localhost:" + PORT + "/qaSingleSeries/query/measurements");
    post.setHeader("Content-Type", "application/json");
    post.setEntity(new StringEntity("{\"target\":\"cpu\"}"));
    response = TestUtils.makeRequest(post);
    assertEquals(200, response.getStatusLine().getStatusCode());
    post = new HttpPost("http://localhost:" + PORT + "/qaSingleSeries/query/measurements");
    post.setHeader("Content-Type", "application/json");
    post.setEntity(new StringEntity("{\"target\":\"cpu.*\"}"));
    response = TestUtils.makeRequest(post);
    assertEquals(200, response.getStatusLine().getStatusCode());
    String res = EntityUtils.toString(response.getEntity());
    List<String> fromJson = new Gson().fromJson(res, List.class);
    assertEquals(1, fromJson.size());
    post = new HttpPost("http://localhost:" + PORT + "/qaSingleSeries/query/measurements");
    post.setHeader("Content-Type", "application/json");
    post.setEntity(new StringEntity("{\"target\":\".*\"}"));
    response = TestUtils.makeRequest(post);
    assertEquals(200, response.getStatusLine().getStatusCode());
    res = EntityUtils.toString(response.getEntity());
    fromJson = new Gson().fromJson(res, List.class);
    assertEquals(1, fromJson.size());
}

Example 12 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestMeasurement method testCompactionEmptyLineValidation.

@Test
public void testCompactionEmptyLineValidation() throws IOException {
    final long ts = 1484788896586L;
    List<Tag> tags = Arrays.asList(Tag.newBuilder().setTagKey("test").setTagValue("1").build(), Tag.newBuilder().setTagKey("test").setTagValue("2").build());
    conf.put(DiskMalloc.CONF_MEASUREMENT_FILE_MAX, String.valueOf(2 * 1024 * 1024));
    conf.put("malloc.ptrfile.increment", String.valueOf(256));
    conf.put("compaction.ratio", "1.2");
    measurement.configure(conf, null, 1024, DBNAME, "m1", indexDir, dataDir, metadata, bgTaskPool);
    int LIMIT = 34500;
    for (int i = 0; i < LIMIT; i++) {
        measurement.addPointWithLocking(build("value1", tags, ts + i * 100, 1.2), false);
    }
    // measurement.collectGarbage(null);
    System.err.println("Gc complete");
    measurement.compact();
    measurement.getSeriesList().iterator().next();
    measurement.close();
    measurement.configure(conf, null, 4096, DBNAME, "m1", indexDir, dataDir, metadata, bgTaskPool);
}

Example 13 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestMeasurement method testGCMemoryMapFree.

@Test
public void testGCMemoryMapFree() throws IOException, InterruptedException {
    final long ts = 1484788896586L;
    DiskMalloc.debug = true;
    List<Tag> tags = Arrays.asList(Tag.newBuilder().setTagKey("test").setTagValue("1").build(), Tag.newBuilder().setTagKey("test").setTagValue("2").build());
    Map<String, String> map = new HashMap<>();
    map.put("compression.class", "byzantine");
    map.put("compaction.class", "byzantine");
    map.put(DiskMalloc.CONF_MEASUREMENT_FILE_MAX, String.valueOf(512 * 1024));
    map.put("malloc.file.increment", String.valueOf(256 * 1024));
    map.put("malloc.buf.increment", String.valueOf(1024));
    map.put("default.series.retention.hours", String.valueOf(2));
    measurement.configure(map, null, 512, DBNAME, "m1", indexDir, dataDir, metadata, bgTaskPool);
    int LIMIT = 20000;
    for (int i = 0; i < LIMIT; i++) {
        for (int k = 0; k < 2; k++) {
            measurement.addPointWithLocking(build("value" + k, tags, ts + i * 10000, i * 1.2), false);
        }
    }
    Series s = measurement.getOrCreateSeries(tags, false);
    // System.out.println(measurement.getOrCreateSeriesFieldMap("value0", tags, 512,
    // false,
    // map).getBucketRawMap().size());
    measurement.collectGarbage(null);
    for (int k = 0; k < 2; k++) {
        List<DataPoint> dps = s.queryDataPoints(measurement, Arrays.asList("value" + k), ts, ts + LIMIT * 10000, null).get("value" + k);
        assertEquals(10032, dps.size());
    }
    System.gc();
    Thread.sleep(200);
    for (int k = 0; k < 2; k++) {
        List<DataPoint> dps = s.queryDataPoints(measurement, Arrays.asList("value" + k), ts, ts + LIMIT * 10000, null).get("value" + k);
        assertEquals(10032, dps.size());
    }
    measurement.close();
}

Example 14 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestMeasurement method testCompaction.

@Test
public void testCompaction() throws IOException {
    final long ts = 1484788896586L;
    List<Tag> tags = Arrays.asList(Tag.newBuilder().setTagKey("test").setTagValue("1").build(), Tag.newBuilder().setTagKey("test2").setTagValue("2").build());
    conf.put(DiskMalloc.CONF_MEASUREMENT_FILE_MAX, String.valueOf(2 * 1024 * 1024));
    conf.put("malloc.ptrfile.increment", String.valueOf(1024));
    TimeField.compactionRatio = 1.2;
    ValueField.compactionRatio = 1.2;
    measurement.configure(conf, null, 1024, DBNAME, "m4", indexDir, dataDir, metadata, bgTaskPool);
    int LIMIT = 12000;
    for (int i = 0; i < LIMIT; i++) {
        measurement.addPointWithLocking(build("value1", tags, ts + i, 1.2 * i), false);
    }
    assertEquals(1, measurement.getSeriesList().size());
    Series series = measurement.getSeriesList().iterator().next();
    System.out.println("Series:" + series);
    assertEquals(1, series.getBucketMap().size());
    assertEquals(5, MiscUtils.bucketCounter(series));
    assertEquals(2, series.getBucketMap().entrySet().iterator().next().getValue().size());
    // int maxDp = series.getBucketFieldMap().values().stream().flatMap(v ->
    // v.stream()).mapToInt(l -> l.getCount())
    // .max().getAsInt();
    // check and read datapoint count before
    Series seriesField = measurement.getSeriesField(tags);
    List<DataPoint> queryDataPoints = seriesField.queryDataPoints(measurement, Arrays.asList("value1"), ts, ts + LIMIT + 1, null).get("value1");
    assertEquals(LIMIT, queryDataPoints.size());
    for (int i = 0; i < LIMIT; i++) {
        DataPoint dp = queryDataPoints.get(i);
        assertEquals(ts + i, dp.getTimestamp());
        assertEquals(i * 1.2, dp.getValue(), 0.01);
    }
    measurement.compact();
    assertEquals(3, MiscUtils.bucketCounter(series));
    assertEquals(2, series.getBucketMap().entrySet().iterator().next().getValue().size());
    // assertTrue(maxDp <= series.getBucketFieldMap().values().stream().flatMap(v ->
    // v.stream())
    // .mapToInt(l -> l.getCount()).max().getAsInt());
    // validate query after compaction
    seriesField = measurement.getSeriesField(tags);
    queryDataPoints = seriesField.queryDataPoints(measurement, Arrays.asList("value1"), ts, ts + LIMIT + 1, null).get("value1");
    assertEquals(LIMIT, queryDataPoints.size());
    for (int i = 0; i < LIMIT; i++) {
        DataPoint dp = queryDataPoints.get(i);
        assertEquals(ts + i, dp.getTimestamp());
        assertEquals(i * 1.2, dp.getValue(), 0.01);
    }
    measurement.close();
}

Example 15 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class MeasurementOpsApi method createSeries.

@Path("/series")
@PUT
@Consumes({ MediaType.APPLICATION_JSON })
public void createSeries(@PathParam(DatabaseOpsApi.DB_NAME) String dbName, @PathParam(MEASUREMENT) String measurementName, String seriesConfig) {
    Gson gson = new Gson();
    JsonObject series = gson.fromJson(seriesConfig, JsonObject.class);
    List<Tag> tags = new ArrayList<>();
    for (JsonElement jsonElement : series.get("tags").getAsJsonArray()) {
        String tagStr = jsonElement.getAsString();
        String[] splits = tagStr.split(Measurement.TAG_KV_SEPARATOR);
        tags.add(Tag.newBuilder().setTagKey(splits[0]).setTagValue(splits[1]).build());
    }
    try {
        Measurement m = engine.getOrCreateMeasurement(dbName, measurementName);
        m.getOrCreateSeries(tags, false);
    } catch (IOException e) {
        throw new InternalServerErrorException(e);
    }
}