Examples with Measurement com.srotya.sidewinder.core.storage.Measurement used on opensource projects

Example 16 with Measurement

use of com.srotya.sidewinder.core.storage.Measurement in project sidewinder by srotya.

the class TestSetTagIndex method testTagIndexPerformance.

// @Test
public void testTagIndexPerformance() throws IOException, InterruptedException {
    MiscUtils.delete(new File("target/perf/index-dir"));
    MiscUtils.delete(new File("target/perf/data-dir"));
    DiskStorageEngine engine = new DiskStorageEngine();
    HashMap<String, String> conf = new HashMap<>();
    conf.put("index.dir", "target/perf/index-dir");
    conf.put("data.dir", "target/perf/data-dir");
    engine.configure(conf, Executors.newScheduledThreadPool(1, new BackgrounThreadFactory("bgt")));
    final long ms = System.currentTimeMillis();
    ExecutorService es = Executors.newCachedThreadPool();
    for (int k = 0; k < 6; k++) {
        es.submit(() -> {
            for (int i = 0; i < 200_000_000; i++) {
                try {
                    engine.getOrCreateTimeSeries("db1", "m1", "v10", Arrays.asList(String.valueOf(i % 10_000), "test=" + "asdasdasd" + String.valueOf(i % 5), "test2=" + String.valueOf(i % 5), "goliath=" + String.valueOf(i % 100_000), "goliath2=" + String.valueOf(i % 1_500)), 4096, true);
                } catch (IOException e) {
                    e.printStackTrace();
                }
                if (i % 1_000_000 == 0) {
                    System.out.println(i + "  " + (System.currentTimeMillis() - ms) / 1000);
                }
            }
        });
    }
    es.shutdown();
    es.awaitTermination(1000, TimeUnit.SECONDS);
    System.err.println("Index time:" + (System.currentTimeMillis() - ms));
    Map<String, Map<String, Measurement>> index = engine.getMeasurementMap();
    assertEquals(1, index.size());
    Entry<String, Map<String, Measurement>> next = index.entrySet().iterator().next();
    assertEquals("db1", next.getKey());
    Entry<String, Measurement> itr = next.getValue().entrySet().iterator().next();
    assertEquals("m1", itr.getKey());
}

Example 17 with Measurement

use of com.srotya.sidewinder.core.storage.Measurement in project sidewinder by srotya.

the class DiskStorageEngine method loadDatabases.

protected void loadDatabases() throws IOException {
    for (String dataDir : dataDirs) {
        File mdDir = new File(dataDir);
        if (!mdDir.exists()) {
            return;
        }
        File[] dbs = mdDir.listFiles();
        for (File db : dbs) {
            if (!db.isDirectory()) {
                continue;
            }
            Map<String, Measurement> measurementMap = new ConcurrentHashMap<>();
            String dbName = db.getName();
            databaseMap.put(dbName, measurementMap);
            DBMetadata metadata = readMetadata(dbName);
            dbMetadataMap.put(dbName, metadata);
            logger.info("Loading database:" + dbName);
            loadMeasurements(dbName, measurementMap, metadata);
        }
    }
}

Example 18 with Measurement

use of com.srotya.sidewinder.core.storage.Measurement in project sidewinder by srotya.

the class DiskStorageEngine method getOrCreateDatabase.

@Override
public Map<String, Measurement> getOrCreateDatabase(String dbName) throws IOException {
    Map<String, Measurement> measurementMap = databaseMap.get(dbName);
    if (measurementMap == null) {
        synchronized (databaseMap) {
            if ((measurementMap = databaseMap.get(dbName)) == null) {
                measurementMap = new ConcurrentHashMap<>();
                databaseMap.put(dbName, measurementMap);
                createDatabaseDirectory(dbName);
                DBMetadata metadata = new DBMetadata();
                metadata.setRetentionHours(defaultRetentionHours);
                dbMetadataMap.put(dbName, metadata);
                saveDBMetadata(dbName, metadata);
                logger.info("Created new database:" + dbName + "\t with retention period:" + defaultRetentionHours + " hours");
                metricsDbCounter.inc();
            }
        }
    }
    return measurementMap;
}

Example 19 with Measurement

use of com.srotya.sidewinder.core.storage.Measurement in project sidewinder by srotya.

the class DiskStorageEngine method loadMeasurements.

protected void loadMeasurements(String dbName, Map<String, Measurement> measurementMap, DBMetadata metadata) throws IOException {
    File file = new File(dbDirectoryPath(dbName));
    if (!file.exists() || file.listFiles() == null) {
        return;
    }
    List<Future<?>> futures = new ArrayList<>();
    for (File measurementMdFile : file.listFiles()) {
        if (!measurementMdFile.isDirectory()) {
            continue;
        }
        String measurementName = measurementMdFile.getName();
        Measurement measurement = new PersistentMeasurement();
        measurementMap.put(measurementName, measurement);
        logger.info("Loading measurements:" + measurementName);
        futures.add(bgTaskPool.submit(() -> {
            try {
                measurement.configure(conf, this, dbName, measurementName, dbIndexPath(dbName), dbDirectoryPath(dbName), metadata, bgTaskPool);
            } catch (IOException e) {
                logger.log(Level.SEVERE, "Error recovering measurement:" + measurementName, e);
            }
        }));
    }
    for (Future<?> future : futures) {
        try {
            future.get();
        } catch (InterruptedException | ExecutionException e) {
            logger.log(Level.SEVERE, "Error future get recovering measurement, db:" + dbName, e);
        }
    }
}

Example 20 with Measurement

use of com.srotya.sidewinder.core.storage.Measurement in project sidewinder by srotya.

the class DiskStorageEngine method dropDatabase.

@Override
public void dropDatabase(String dbName) throws Exception {
    synchronized (databaseMap) {
        Map<String, Measurement> remove = databaseMap.remove(dbName);
        for (Measurement measurement : remove.values()) {
            measurement.close();
        }
        boolean result = MiscUtils.delete(new File(dbDirectoryPath(dbName)));
        if (!result) {
            throw new Exception("Database(" + dbName + ") deletion(data) failed due file deletion issues");
        }
        result = MiscUtils.delete(new File(dbIndexPath(dbName)));
        if (!result) {
            throw new Exception("Database(" + dbName + ") deletion(index) failed due file deletion issues");
        }
        metricsDbCounter.dec();
        logger.info("Database(" + dbName + ") deleted");
    }
}