Examples with Tag com.srotya.sidewinder.core.rpc.Tag used on opensource projects

Example 31 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestStorageEngine method testConcurrentOperations.

@Test
public void testConcurrentOperations() throws Exception {
    engine.configure(conf, bgTasks);
    final long ts = System.currentTimeMillis();
    ExecutorService es = Executors.newFixedThreadPool(2, new BackgrounThreadFactory("wr1"));
    String measurementName = "mmm2";
    String valueFieldName = "v1";
    String dbName = "db9";
    List<Tag> tagd = Arrays.asList(Tag.newBuilder().setTagKey("h").setTagValue("1").build());
    for (int k = 0; k < 2; k++) {
        final int p = k;
        es.submit(() -> {
            long t = ts + p;
            for (int i = 0; i < 100; i++) {
                Point dp = MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, tagd, t + i * 1000, i);
                try {
                    engine.writeDataPointWithLock(dp, false);
                } catch (Exception e) {
                    e.printStackTrace();
                    throw new RuntimeException(e);
                }
            }
            System.err.println("Completed writes:" + 100 + " data points");
        });
    }
    es.shutdown();
    es.awaitTermination(100, TimeUnit.SECONDS);
    assertEquals(1, engine.getAllMeasurementsForDb(dbName).size());
    assertEquals(1, engine.getMeasurementMap().size());
    try {
        Series timeSeries = engine.getTimeSeries(dbName, measurementName, valueFieldName, tagd);
        assertNotNull(timeSeries);
    } catch (ItemNotFoundException e) {
        fail("Time series must exist");
    }
    List<SeriesOutput> queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, ts, ts + 220 * 1000, null);
    assertEquals(1, queryDataPoints.size());
    SeriesOutput next = queryDataPoints.iterator().next();
    assertEquals(200, next.getDataPoints().size());
}

Example 32 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestDiskStorageEngine method testMultipleDrives2.

@Test
public void testMultipleDrives2() throws ItemNotFoundException, IOException {
    StorageEngine engine = new DiskStorageEngine();
    HashMap<String, String> map = new HashMap<>();
    MiscUtils.delete(new File("targer/db10221/"));
    map.put("index.dir", "target/db10221/index");
    map.put("data.dir", "target/db10221/data1,target/db10221/data2");
    List<Tag> tagd = Arrays.asList(Tag.newBuilder().setTagKey("t").setTagValue("e").build());
    try {
        engine.configure(map, bgTasks);
    } catch (IOException e) {
        e.printStackTrace();
        fail("No IOException should be thrown");
    }
    long ts = System.currentTimeMillis();
    try {
        for (int i = 0; i < 10; i++) {
            engine.writeDataPointWithLock(MiscUtils.buildDataPoint("test" + i, "ss", "value", tagd, ts, 2.2), false);
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail("Engine is initialized, no IO Exception should be thrown:" + e.getMessage());
    }
    assertEquals(5, new File("target/db10221/data1").listFiles().length);
    assertEquals(5, new File("target/db10221/data2").listFiles().length);
    engine.shutdown();
}

Example 33 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestDiskStorageEngine method testQueryDataPointsRecovery.

@Test
public void testQueryDataPointsRecovery() throws Exception {
    try {
        List<Tag> tagd = Arrays.asList(Tag.newBuilder().setTagKey("test").setTagValue("e").build());
        DiskStorageEngine engine = new DiskStorageEngine();
        File file = new File("target/db201/");
        MiscUtils.delete(file);
        MiscUtils.ls(file);
        Map<String, String> map = new HashMap<>();
        map.put("index.dir", "target/db201/index");
        map.put("data.dir", "target/db201/data");
        engine.configure(map, bgTasks);
        long ts = System.currentTimeMillis();
        Map<String, Measurement> db = engine.getOrCreateDatabase("test3");
        assertEquals(0, db.size());
        engine.writeDataPointWithLock(MiscUtils.buildDataPoint("test3", "cpu", "value", tagd, ts, 1), false);
        engine.writeDataPointWithLock(MiscUtils.buildDataPoint("test3", "cpu", "value", tagd, ts + (400 * 60000), 4), false);
        Measurement measurement = engine.getOrCreateMeasurement("test3", "cpu");
        assertEquals(1, measurement.getSeriesKeys().size());
        MiscUtils.ls(file);
        engine = new DiskStorageEngine();
        engine.configure(map, bgTasks);
        assertTrue(!engine.isMeasurementFieldFP("test3", "cpu", "value"));
        List<SeriesOutput> queryDataPoints = engine.queryDataPoints("test3", "cpu", "value", ts, ts + (400 * 60000), null, null);
        try {
            engine.isMeasurementFieldFP("test3", "test", "test");
            fail("Measurement should not exist");
        } catch (Exception e) {
        }
        assertEquals(1, queryDataPoints.size());
        assertEquals(2, queryDataPoints.iterator().next().getDataPoints().size());
        assertEquals(ts, queryDataPoints.iterator().next().getDataPoints().get(0).getTimestamp());
        assertEquals(ts + (400 * 60000), queryDataPoints.iterator().next().getDataPoints().get(1).getTimestamp());
        TagFilter filter = MiscUtils.buildTagFilter("test=e");
        queryDataPoints = engine.queryDataPoints("test3", "cpu", "value", ts, ts + (400 * 60000), filter, null);
        assertEquals(1, queryDataPoints.size());
        assertEquals(2, queryDataPoints.iterator().next().getDataPoints().size());
        assertEquals(ts, queryDataPoints.iterator().next().getDataPoints().get(0).getTimestamp());
        assertEquals(ts + (400 * 60000), queryDataPoints.iterator().next().getDataPoints().get(1).getTimestamp());
        filter = MiscUtils.buildTagFilter("test=2");
        queryDataPoints = engine.queryDataPoints("test3", "cpu", "value", ts, ts + (400 * 60000), filter, null);
        assertEquals(0, queryDataPoints.size());
        try {
            engine.dropDatabase("test3");
        } catch (Exception e) {
            e.printStackTrace();
            fail("Database delete must succeed");
        }
        assertTrue(!new File("target/db201/data/test3").exists());
        assertEquals(0, engine.getOrCreateMeasurement("test3", "cpu").getSeriesKeys().size());
        engine.shutdown();
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
}

Example 34 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestDiskStorageEngine method testMultipleDrives.

@Test
public void testMultipleDrives() throws ItemNotFoundException, IOException {
    StorageEngine engine = new DiskStorageEngine();
    HashMap<String, String> map = new HashMap<>();
    MiscUtils.delete(new File("targer/db10221/"));
    map.put("index.dir", "target/db10221/index");
    map.put("data.dir", "target/db10221/data1, target/db10221/data2");
    List<Tag> tagd = Arrays.asList(Tag.newBuilder().setTagKey("t").setTagValue("e").build());
    try {
        engine.configure(map, bgTasks);
    } catch (IOException e) {
        e.printStackTrace();
        fail("No IOException should be thrown");
    }
    long ts = System.currentTimeMillis();
    try {
        for (int i = 0; i < 10; i++) {
            engine.writeDataPointWithLock(MiscUtils.buildDataPoint("test" + i, "ss", "value", tagd, ts, 2.2), false);
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail("Engine is initialized, no IO Exception should be thrown:" + e.getMessage());
    }
    assertEquals(5, new File("target/db10221/data1").listFiles().length);
    assertEquals(5, new File("target/db10221/data2").listFiles().length);
    engine.shutdown();
}

Example 35 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class QAInMemoryByzantineDefaultsAuthenticated method testSingleSeriesWrites.

@Test
public void testSingleSeriesWrites() throws KeyManagementException, ClientProtocolException, NoSuchAlgorithmException, KeyStoreException, MalformedURLException, IOException, ParseException, InterruptedException {
    long sts = 1497720452566L;
    SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
    format.setTimeZone(TimeZone.getTimeZone("utc"));
    String payload = "{\"panelId\":2,\"range\":{\"from\":\"%s\",\"to\":\"%s\",\"raw\":{\"from\":\"now-5m\",\"to\":\"now\"}},\"rangeRaw\":{\"from\":\"now-5m\",\"to\":\"now\"},\"interval\":\"200ms\",\"intervalMs\":200,\"targets\":[{\"target\":\"cpu\",\"filters\":[],\"aggregator\":{\"args\":[{\"index\":0,\"type\":\"int\",\"value\":\"20\"}],\"name\":\"none\",\"unit\":\"secs\"},\"field\":\"value\",\"refId\":\"A\",\"type\":\"timeserie\"}],\"format\":\"json\",\"maxDataPoints\":1280}";
    HttpPost post = new HttpPost("http://localhost:" + PORT + "/influx?db=qaSingleSeries");
    CloseableHttpResponse response = TestUtils.makeRequestAuthenticated(post, provider);
    assertEquals(400, response.getStatusLine().getStatusCode());
    post = new HttpPost("http://localhost:" + PORT + "/influx?db=qaSingleSeries");
    post.setEntity(new StringEntity("cpu,host=server01,region=uswest value=1i 1497720452566000000\n" + "cpu,host=server02,region=uswest value=1i 1497720452566000000\n" + "cpu,host=server03,region=uswest value=1i 1497720452566000000\n" + "cpu,host=server01,region=uswest value=1i 1497720453566000000\n" + "cpu,host=server02,region=uswest value=1i 1497720453566000000\n" + "cpu,host=server03,region=uswest value=1i 1497720453566000000"));
    response = TestUtils.makeRequestAuthenticated(post, provider);
    assertEquals(204, response.getStatusLine().getStatusCode());
    post = new HttpPost("http://localhost:" + PORT + "/qaSingleSeries/query/measurements");
    post.setHeader("Content-Type", "application/json");
    response = TestUtils.makeRequestAuthenticated(post, provider);
    Gson gson = new Gson();
    JsonArray ary = gson.fromJson(EntityUtils.toString(response.getEntity()), JsonArray.class);
    assertEquals("cpu", ary.get(0).getAsString());
    post = new HttpPost("http://localhost:" + PORT + "/qaSingleSeries/query/tags");
    post.setHeader("Content-Type", "application/json");
    post.setEntity(new StringEntity("{ \"target\":\"cpu\" }"));
    response = TestUtils.makeRequestAuthenticated(post, provider);
    ary = gson.fromJson(EntityUtils.toString(response.getEntity()), JsonArray.class);
    assertEquals(2, ary.size());
    post = new HttpPost("http://localhost:" + PORT + "/qaSingleSeries/query");
    post.setHeader("Content-Type", "application/json");
    post.setEntity(new StringEntity(String.format(payload, format.format(new Date(sts - 60_000)), format.format(new Date(sts + 60_000)))));
    response = TestUtils.makeRequestAuthenticated(post, provider);
    ary = gson.fromJson(EntityUtils.toString(response.getEntity()), JsonArray.class);
    assertEquals(3, ary.size());
    int i = 0;
    for (JsonElement ele : ary) {
        i += ele.getAsJsonObject().get("datapoints").getAsJsonArray().size();
    }
    assertEquals(6, i);
    response = TestUtils.makeRequestAuthenticated(new HttpGet("http://localhost:" + PORT + "/databases/qaSingleSeries/measurements/cpu/fields/value?startTime=" + (sts - 2000) + "&endTime=" + (sts + 2000)), provider);
    ary = gson.fromJson(EntityUtils.toString(response.getEntity()), JsonArray.class);
    Set<Tag> tag = new HashSet<>(Arrays.asList(Tag.newBuilder().setTagKey("host").setTagValue("server02").build(), Tag.newBuilder().setTagKey("host").setTagValue("server01").build(), Tag.newBuilder().setTagKey("host").setTagValue("server03").build(), Tag.newBuilder().setTagKey("region").setTagValue("uswest").build()));
    Iterator<JsonElement> itr = ary.iterator();
    i = 0;
    while (itr.hasNext()) {
        JsonObject obj = itr.next().getAsJsonObject();
        assertEquals("cpu", obj.get("measurementName").getAsString());
        assertEquals("value", obj.get("valueFieldName").getAsString());
        ary = obj.get("tags").getAsJsonArray();
        for (JsonElement ele : ary) {
            Tag tagObj = gson.fromJson(ele, Tag.class);
            assertTrue(ele + " ", tag.contains(tagObj));
        }
        i += obj.get("dataPoints").getAsJsonArray().size();
        JsonArray ary2 = obj.get("dataPoints").getAsJsonArray();
        for (int j = 0; j < ary2.size(); j++) {
            JsonElement ele = ary2.get(j);
            assertEquals(1, ele.getAsJsonObject().get("value").getAsInt());
            long ts = 1497720452566L + j * 1000;
            assertEquals(ts, ele.getAsJsonObject().get("timestamp").getAsLong());
        }
    }
    assertEquals(6, i);
}