use of com.srotya.sidewinder.core.storage.TimeSeries in project sidewinder by srotya.
the class CalciteTest method main.
public static void main(String[] args) throws IOException, ClassNotFoundException, SQLException {
ScheduledExecutorService bgt = Executors.newScheduledThreadPool(1, new BackgrounThreadFactory("sidewinderbg-tasks"));
MemStorageEngine engine = new MemStorageEngine();
engine.configure(new HashMap<>(), bgt);
SqlApi api = new SqlApi(engine);
api.initCalcite();
long ts = System.currentTimeMillis();
TimeSeries s = engine.getOrCreateTimeSeries("db1", "m1", "v1", Arrays.asList("t=1", "p=2"), 1024, false);
for (int i = 0; i < 100; i++) {
s.addDataPoint(TimeUnit.MILLISECONDS, ts + i * 1000, i);
}
String queryResults = api.queryResults("db1", "select * from db1.m1 where time_stamp>1519945488603");
System.out.println(queryResults);
}
use of com.srotya.sidewinder.core.storage.TimeSeries in project sidewinder by srotya.
the class TestPersistentMeasurement method testMeasurementRecovery.
@Test
public void testMeasurementRecovery() throws IOException {
MiscUtils.delete(new File("target/db141/"));
PersistentMeasurement m = new PersistentMeasurement();
m.configure(conf, engine, DBNAME, "m1", "target/db141/index", "target/db141/data", metadata, bgTaskPool);
TimeSeries ts = m.getOrCreateTimeSeries("vf1", Arrays.asList("t=1", "t=2"), 4096, false, conf);
long t = System.currentTimeMillis();
for (int i = 0; i < 100; i++) {
ts.addDataPoint(TimeUnit.MILLISECONDS, t + i * 1000, i);
}
List<DataPoint> dps = ts.queryDataPoints("vf1", t, t + 1000 * 100, null);
assertEquals(100, dps.size());
for (int i = 0; i < 100; i++) {
DataPoint dp = dps.get(i);
assertEquals(t + i * 1000, dp.getTimestamp());
assertEquals(i, dp.getLongValue());
}
List<Series> resultMap = new ArrayList<>();
m.queryDataPoints("vf1", t, t + 1000 * 100, null, null, resultMap);
assertEquals(1, resultMap.size());
Series next = resultMap.iterator().next();
for (int i = 0; i < next.getDataPoints().size(); i++) {
DataPoint dp = next.getDataPoints().get(i);
assertEquals(t + i * 1000, dp.getTimestamp());
assertEquals(i, dp.getLongValue());
}
LinkedHashMap<Reader, Boolean> readers = new LinkedHashMap<>();
m.queryReaders("vf1", t, t + 1000 * 100, readers);
for (Reader reader : readers.keySet()) {
assertEquals(100, reader.getPairCount());
}
m.close();
}
use of com.srotya.sidewinder.core.storage.TimeSeries in project sidewinder by srotya.
the class TestPersistentMeasurement method testLinearizabilityWithRollOverBucket.
@Test
public void testLinearizabilityWithRollOverBucket() throws IOException, InterruptedException {
for (int p = 0; p < 2; p++) {
final int LIMIT = 10000;
MiscUtils.delete(new File("target/db135/"));
final long t1 = 1497720452566L;
Measurement m = new PersistentMeasurement();
m.configure(conf, engine, DBNAME, "m1", "target/db135/index", "target/db135/data", metadata, bgTaskPool);
ExecutorService es = Executors.newFixedThreadPool(2, new BackgrounThreadFactory("tlinear2"));
AtomicBoolean wait = new AtomicBoolean(false);
for (int i = 0; i < 2; i++) {
final int th = i;
es.submit(() -> {
while (!wait.get()) {
try {
Thread.sleep(1);
} catch (InterruptedException e) {
}
}
long t = t1 + th * 3;
for (int j = 0; j < LIMIT; j++) {
try {
TimeSeries ts = m.getOrCreateTimeSeries("vf1", Arrays.asList("t=1", "t=2"), 4096, false, conf);
long timestamp = t + j * 1000;
ts.addDataPoint(TimeUnit.MILLISECONDS, timestamp, j);
} catch (Exception e) {
e.printStackTrace();
}
}
});
}
es.shutdown();
wait.set(true);
es.awaitTermination(10, TimeUnit.SECONDS);
TimeSeries ts = m.getOrCreateTimeSeries("vf1", Arrays.asList("t=1", "t=2"), 4096, false, conf);
List<DataPoint> dps = ts.queryDataPoints("vf1", t1 - 100, t1 + 1000_0000, null);
assertEquals(LIMIT * 2, dps.size(), 10);
m.close();
}
}
use of com.srotya.sidewinder.core.storage.TimeSeries in project sidewinder by srotya.
the class TestPersistentMeasurement method testDataPointsQuery.
@Test
public void testDataPointsQuery() throws Exception {
long ts = System.currentTimeMillis();
MiscUtils.delete(new File("target/db41/"));
List<String> tags = Arrays.asList("test=1", "test=2");
PersistentMeasurement m = new PersistentMeasurement();
Map<String, String> map = new HashMap<>();
map.put("disk.compression.class", ByzantineWriter.class.getName());
map.put("malloc.file.max", String.valueOf(2 * 1024 * 1024));
m.configure(map, null, DBNAME, "m1", "target/db41/index", "target/db41/data", metadata, bgTaskPool);
int LIMIT = 1000;
for (int i = 0; i < LIMIT; i++) {
TimeSeries t = m.getOrCreateTimeSeries("value1", tags, 4096, false, map);
t.addDataPoint(TimeUnit.MILLISECONDS, ts + i * 1000, 1L);
}
for (int i = 0; i < LIMIT; i++) {
TimeSeries t = m.getOrCreateTimeSeries("value2", tags, 4096, false, map);
t.addDataPoint(TimeUnit.MILLISECONDS, ts + i * 1000, 1L);
}
List<Series> resultMap = new ArrayList<>();
m.queryDataPoints("value.*$", ts, ts + 1000 * LIMIT, null, null, resultMap);
assertEquals(2, resultMap.size());
for (Series s : resultMap) {
for (int i = 0; i < s.getDataPoints().size(); i++) {
DataPoint dataPoint = s.getDataPoints().get(i);
assertEquals(ts + i * 1000, dataPoint.getTimestamp());
assertEquals(1L, dataPoint.getLongValue());
}
}
List<List<Tag>> tagsResult = m.getTagsForMeasurement();
Collections.sort(tags);
for (List<Tag> list : tagsResult) {
Set<Tag> hashSet = new HashSet<>(list);
for (int i = 0; i < tags.size(); i++) {
String tag = tags.get(i);
String[] split = tag.split("=");
assertTrue(hashSet.contains(new Tag(split[0], split[1])));
}
}
try {
tagsResult = m.getTagsForMeasurement();
} catch (IOException e) {
}
m.close();
}
use of com.srotya.sidewinder.core.storage.TimeSeries in project sidewinder by srotya.
the class TestPersistentMeasurement method testMemoryMapFree.
@Test
public void testMemoryMapFree() throws IOException, InterruptedException {
final long ts = 1484788896586L;
DiskMalloc.debug = true;
MiscUtils.delete(new File("target/db46/"));
List<String> tags = Arrays.asList("test=1", "test=2");
PersistentMeasurement m = new PersistentMeasurement();
Map<String, String> map = new HashMap<>();
map.put("compression.class", "byzantine");
map.put("compaction.class", "byzantine");
map.put("malloc.file.max", String.valueOf(512 * 1024));
map.put("malloc.file.increment", String.valueOf(256 * 1024));
map.put("malloc.buf.increment", String.valueOf(1024));
map.put("default.series.retention.hours", String.valueOf(2));
map.put("compaction.ratio", "1.2");
map.put("compaction.enabled", "true");
m.configure(map, null, DBNAME, "m1", "target/db46/index", "target/db46/data", metadata, bgTaskPool);
int LIMIT = 20000;
for (int i = 0; i < LIMIT; i++) {
for (int k = 0; k < 2; k++) {
TimeSeries t = m.getOrCreateTimeSeries("value" + k, tags, 512, false, map);
t.addDataPoint(TimeUnit.MILLISECONDS, ts + i * 10000, i * 1.2);
}
}
System.out.println(m.getOrCreateTimeSeries("value0", tags, 512, false, map).getBucketRawMap().size());
m.collectGarbage(null);
for (int k = 0; k < 2; k++) {
TimeSeries t = m.getOrCreateTimeSeries("value" + k, tags, 512, false, map);
List<DataPoint> dps = t.queryDataPoints("", ts, ts + LIMIT * 10000, null);
assertEquals(10032, dps.size());
}
System.gc();
Thread.sleep(200);
for (int k = 0; k < 2; k++) {
TimeSeries t = m.getOrCreateTimeSeries("value" + k, tags, 512, false, map);
List<DataPoint> dps = t.queryDataPoints("", ts, ts + LIMIT * 10000, null);
assertEquals(10032, dps.size());
}
m.close();
}
Aggregations