Examples with BackgrounThreadFactory com.srotya.sidewinder.core.utils.BackgrounThreadFactory used on opensource projects

Example 1 with BackgrounThreadFactory

use of com.srotya.sidewinder.core.utils.BackgrounThreadFactory in project sidewinder by srotya.

the class CalciteTest method main.

public static void main(String[] args) throws IOException, ClassNotFoundException, SQLException {
    ScheduledExecutorService bgt = Executors.newScheduledThreadPool(1, new BackgrounThreadFactory("sidewinderbg-tasks"));
    MemStorageEngine engine = new MemStorageEngine();
    engine.configure(new HashMap<>(), bgt);
    SqlApi api = new SqlApi(engine);
    api.initCalcite();
    long ts = System.currentTimeMillis();
    TimeSeries s = engine.getOrCreateTimeSeries("db1", "m1", "v1", Arrays.asList("t=1", "p=2"), 1024, false);
    for (int i = 0; i < 100; i++) {
        s.addDataPoint(TimeUnit.MILLISECONDS, ts + i * 1000, i);
    }
    String queryResults = api.queryResults("db1", "select * from db1.m1 where time_stamp>1519945488603");
    System.out.println(queryResults);
}

Example 2 with BackgrounThreadFactory

use of com.srotya.sidewinder.core.utils.BackgrounThreadFactory in project sidewinder by srotya.

the class SidewinderServer method run.

@Override
public void run(SidewinderConfig config, Environment env) throws Exception {
    Map<String, String> conf = new HashMap<>();
    overloadProperties(config, conf);
    ScheduledExecutorService bgTasks = Executors.newScheduledThreadPool(Integer.parseInt(conf.getOrDefault(ConfigConstants.BG_THREAD_COUNT, "2")), new BackgrounThreadFactory("sidewinderbg-tasks"));
    initializeStorageEngine(conf, bgTasks);
    enableMonitoring(bgTasks);
    registerWebAPIs(env, conf, bgTasks);
    checkAndEnableIngesters(conf, env);
    checkAndRegisterFunctions(conf);
}

Example 3 with BackgrounThreadFactory

use of com.srotya.sidewinder.core.utils.BackgrounThreadFactory in project sidewinder by srotya.

the class TestPersistentMeasurement method testLinearizabilityWithRollOverBucket.

@Test
public void testLinearizabilityWithRollOverBucket() throws IOException, InterruptedException {
    for (int p = 0; p < 2; p++) {
        final int LIMIT = 10000;
        MiscUtils.delete(new File("target/db135/"));
        final long t1 = 1497720452566L;
        Measurement m = new PersistentMeasurement();
        m.configure(conf, engine, DBNAME, "m1", "target/db135/index", "target/db135/data", metadata, bgTaskPool);
        ExecutorService es = Executors.newFixedThreadPool(2, new BackgrounThreadFactory("tlinear2"));
        AtomicBoolean wait = new AtomicBoolean(false);
        for (int i = 0; i < 2; i++) {
            final int th = i;
            es.submit(() -> {
                while (!wait.get()) {
                    try {
                        Thread.sleep(1);
                    } catch (InterruptedException e) {
                    }
                }
                long t = t1 + th * 3;
                for (int j = 0; j < LIMIT; j++) {
                    try {
                        TimeSeries ts = m.getOrCreateTimeSeries("vf1", Arrays.asList("t=1", "t=2"), 4096, false, conf);
                        long timestamp = t + j * 1000;
                        ts.addDataPoint(TimeUnit.MILLISECONDS, timestamp, j);
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            });
        }
        es.shutdown();
        wait.set(true);
        es.awaitTermination(10, TimeUnit.SECONDS);
        TimeSeries ts = m.getOrCreateTimeSeries("vf1", Arrays.asList("t=1", "t=2"), 4096, false, conf);
        List<DataPoint> dps = ts.queryDataPoints("vf1", t1 - 100, t1 + 1000_0000, null);
        assertEquals(LIMIT * 2, dps.size(), 10);
        m.close();
    }
}

Example 4 with BackgrounThreadFactory

use of com.srotya.sidewinder.core.utils.BackgrounThreadFactory in project sidewinder by srotya.

the class TestDiskStorageEngine method testConcurrentOperations.

@Test
public void testConcurrentOperations() throws Exception {
    MiscUtils.delete(new File("target/dst-6/data"));
    final StorageEngine engine = new DiskStorageEngine();
    Map<String, String> conf = new HashMap<>();
    conf.put("data.dir", "target/dst-6/data");
    conf.put("index.dir", "target/dst-6/index");
    engine.configure(conf, bgTasks);
    final long ts = System.currentTimeMillis();
    ExecutorService es = Executors.newFixedThreadPool(2, new BackgrounThreadFactory("wr1"));
    String measurementName = "mmm2";
    String valueFieldName = "v1";
    String dbName = "db9";
    String tag = "h=1";
    for (int k = 0; k < 2; k++) {
        final int p = k;
        es.submit(() -> {
            long t = ts + p;
            for (int i = 0; i < 100; i++) {
                Point dp = MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, Arrays.asList(tag), t + i * 1000, i);
                try {
                    engine.writeDataPoint(dp);
                } catch (Exception e) {
                    e.printStackTrace();
                    throw new RuntimeException(e);
                }
            }
            System.err.println("Completed writes:" + 100 + " data points");
        });
    }
    es.shutdown();
    es.awaitTermination(100, TimeUnit.SECONDS);
    assertEquals(1, engine.getAllMeasurementsForDb(dbName).size());
    assertEquals(1, engine.getMeasurementMap().size());
    try {
        TimeSeries timeSeries = engine.getTimeSeries(dbName, measurementName, valueFieldName, Arrays.asList(tag));
        assertNotNull(timeSeries);
    } catch (ItemNotFoundException e) {
        fail("Time series must exist");
    }
    List<Series> queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, ts, ts + 220 * 1000, null);
    assertEquals(1, queryDataPoints.size());
    Series next = queryDataPoints.iterator().next();
    assertEquals(200, next.getDataPoints().size());
}

Example 5 with BackgrounThreadFactory

use of com.srotya.sidewinder.core.utils.BackgrounThreadFactory in project sidewinder by srotya.

the class TestMappedSetTagIndex method testTagIndexPerformance.

// @Test
public void testTagIndexPerformance() throws IOException, InterruptedException {
    MiscUtils.delete(new File("target/perf/index-dir"));
    MiscUtils.delete(new File("target/perf/data-dir"));
    DiskStorageEngine engine = new DiskStorageEngine();
    HashMap<String, String> conf = new HashMap<>();
    conf.put("index.dir", "target/perf/index-dir");
    conf.put("data.dir", "target/perf/data-dir");
    engine.configure(conf, Executors.newScheduledThreadPool(1, new BackgrounThreadFactory("bgt")));
    final long ms = System.currentTimeMillis();
    ExecutorService es = Executors.newCachedThreadPool();
    for (int k = 0; k < 6; k++) {
        es.submit(() -> {
            for (int i = 0; i < 200_000_000; i++) {
                try {
                    engine.getOrCreateTimeSeries("db1", "m1", "v10", Arrays.asList(String.valueOf(i % 10_000), "test=" + String.valueOf(i % 5), "test2=" + String.valueOf(i % 5), "goliath=" + String.valueOf(i % 10_000), "goliath2=" + String.valueOf(i % 1_500)), 4096, true);
                } catch (IOException e) {
                    e.printStackTrace();
                }
                if (i % 1_000_000 == 0) {
                    System.out.println(i + "\t" + (System.currentTimeMillis() - ms) / 1000);
                }
            }
        });
    }
    es.shutdown();
    es.awaitTermination(1000, TimeUnit.SECONDS);
    System.err.println("Index time:" + (System.currentTimeMillis() - ms));
    Map<String, Map<String, Measurement>> index = engine.getMeasurementMap();
    assertEquals(1, index.size());
    Entry<String, Map<String, Measurement>> next = index.entrySet().iterator().next();
    assertEquals("db1", next.getKey());
    Entry<String, Measurement> itr = next.getValue().entrySet().iterator().next();
    assertEquals("m1", itr.getKey());
    MappedSetTagIndex value = (MappedSetTagIndex) itr.getValue().getTagIndex();
    assertEquals(20000 + 10 + 1500, value.getTagKeys().size());
}