Examples with Measurement com.srotya.sidewinder.core.storage.Measurement used on opensource projects

Example 21 with Measurement

use of com.srotya.sidewinder.core.storage.Measurement in project sidewinder by srotya.

the class DiskStorageEngine method isMeasurementFieldFP.

@Override
public boolean isMeasurementFieldFP(String dbName, String measurementName, String valueFieldName) throws IOException {
    if (!checkIfExists(dbName, measurementName)) {
        throw NOT_FOUND_EXCEPTION;
    }
    Map<String, Measurement> dbMap = getOrCreateDatabase(dbName);
    // check and create measurement map
    Measurement measurement = getOrCreateMeasurement(dbMap, measurementName, dbName);
    for (String entry : measurement.getSeriesKeys()) {
        SeriesFieldMap seriesFromKey = measurement.getSeriesFromKey(entry);
        if (seriesFromKey.get(valueFieldName) != null) {
            return seriesFromKey.get(valueFieldName).isFp();
        }
    }
    throw NOT_FOUND_EXCEPTION;
}

Example 22 with Measurement

use of com.srotya.sidewinder.core.storage.Measurement in project sidewinder by srotya.

the class DiskStorageEngine method getOrCreateTimeSeries.

@Override
public TimeSeries getOrCreateTimeSeries(String dbName, String measurementName, String valueFieldName, List<String> tags, int timeBucketSize, boolean fp) throws IOException {
    // check and create database map
    Map<String, Measurement> dbMap = getOrCreateDatabase(dbName);
    // check and create measurement map
    Measurement measurement = getOrCreateMeasurement(dbMap, measurementName, dbName);
    // check and create timeseries
    return measurement.getOrCreateTimeSeries(valueFieldName, tags, timeBucketSize, fp, conf);
}

Example 23 with Measurement

use of com.srotya.sidewinder.core.storage.Measurement in project sidewinder by srotya.

the class DiskStorageEngine method configure.

@Override
public void configure(Map<String, String> conf, ScheduledExecutorService bgTaskPool) throws IOException {
    this.conf = conf;
    this.bgTaskPool = bgTaskPool;
    this.defaultRetentionHours = Integer.parseInt(conf.getOrDefault(RETENTION_HOURS, String.valueOf(DEFAULT_RETENTION_HOURS)));
    logger.info("Setting default timeseries retention hours policy to:" + defaultRetentionHours);
    dataDirs = MiscUtils.splitAndNormalizeString(conf.getOrDefault(DATA_DIRS, TMP_SIDEWINDER_METADATA));
    baseIndexDirectory = conf.getOrDefault(INDEX_DIR, TMP_SIDEWINDER_INDEX);
    for (String dataDir : dataDirs) {
        new File(dataDir).mkdirs();
    }
    new File(baseIndexDirectory).mkdirs();
    databaseMap = new ConcurrentHashMap<>();
    dbMetadataMap = new ConcurrentHashMap<>();
    try {
        archiver = (Archiver) Class.forName(conf.getOrDefault(ARCHIVER_CLASS, NoneArchiver.class.getName())).newInstance();
        archiver.init(conf);
    } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
        logger.log(Level.SEVERE, "Failed to instantiate archiver", e);
    }
    this.defaultTimebucketSize = Integer.parseInt(conf.getOrDefault(DEFAULT_BUCKET_SIZE, String.valueOf(DEFAULT_TIME_BUCKET_CONSTANT)));
    logger.info("Configuring default time bucket:" + getDefaultTimebucketSize());
    enableMetricsService();
    if (bgTaskPool != null) {
        if (Boolean.parseBoolean(conf.getOrDefault(GC_ENABLED, "true"))) {
            bgTaskPool.scheduleAtFixedRate(() -> {
                for (Entry<String, Map<String, Measurement>> measurementMap : databaseMap.entrySet()) {
                    for (Entry<String, Measurement> measurementEntry : measurementMap.getValue().entrySet()) {
                        Measurement value = measurementEntry.getValue();
                        try {
                            value.collectGarbage(archiver);
                        } catch (Exception e) {
                            logger.log(Level.SEVERE, "Failed collect garbage for measurement:" + value.getMeasurementName(), e);
                        }
                        logger.log(Level.FINE, "Completed Measuremenet GC:" + measurementEntry.getKey());
                    }
                    logger.log(Level.FINE, "Completed DB GC:" + measurementMap.getKey());
                }
            }, Integer.parseInt(conf.getOrDefault(GC_FREQUENCY, DEFAULT_GC_FREQUENCY)), Integer.parseInt(conf.getOrDefault(GC_DELAY, DEFAULT_GC_DELAY)), TimeUnit.SECONDS);
        } else {
            logger.info("WARNING: GC has been disabled, data retention policies will not be honored");
        }
        if (Boolean.parseBoolean(conf.getOrDefault(StorageEngine.COMPACTION_ENABLED, "false"))) {
            logger.info("Compaction is enabled");
            bgTaskPool.scheduleAtFixedRate(() -> {
                for (Entry<String, Map<String, Measurement>> measurementMap : databaseMap.entrySet()) {
                    for (Entry<String, Measurement> measurementEntry : measurementMap.getValue().entrySet()) {
                        Measurement value = measurementEntry.getValue();
                        try {
                            value.compact();
                        } catch (Exception e) {
                            logger.log(Level.SEVERE, "Failed compaction for measurement:" + value.getMeasurementName(), e);
                        }
                    }
                }
            }, Integer.parseInt(conf.getOrDefault(COMPACTION_FREQUENCY, DEFAULT_COMPACTION_FREQUENCY)), Integer.parseInt(conf.getOrDefault(COMPACTION_DELAY, DEFAULT_COMPACTION_DELAY)), TimeUnit.SECONDS);
        } else {
            logger.warning("Compaction is disabled");
        }
    }
    loadDatabases();
}

Example 24 with Measurement

use of com.srotya.sidewinder.core.storage.Measurement in project sidewinder by srotya.

the class MemStorageEngine method getOrCreateDatabase.

@Override
public Map<String, Measurement> getOrCreateDatabase(String dbName) {
    Map<String, Measurement> measurementMap = databaseMap.get(dbName);
    if (measurementMap == null) {
        synchronized (databaseMap) {
            if ((measurementMap = databaseMap.get(dbName)) == null) {
                measurementMap = new ConcurrentHashMap<>();
                databaseMap.put(dbName, measurementMap);
                DBMetadata metadata = new DBMetadata();
                metadata.setRetentionHours(defaultRetentionHours);
                dbMetadataMap.put(dbName, metadata);
                logger.info("Created new database:" + dbName + "\t with retention period:" + defaultRetentionHours + " hours");
                metricsDbCounter.inc();
            }
        }
    }
    return measurementMap;
}

Example 25 with Measurement

use of com.srotya.sidewinder.core.storage.Measurement in project sidewinder by srotya.

the class MemStorageEngine method isMeasurementFieldFP.

@Override
public boolean isMeasurementFieldFP(String dbName, String measurementName, String valueFieldName) throws IOException {
    if (!checkIfExists(dbName, measurementName)) {
        throw NOT_FOUND_EXCEPTION;
    }
    Map<String, Measurement> dbMap = getOrCreateDatabase(dbName);
    // check and create measurement map
    Measurement measurement = getOrCreateMeasurement(dbMap, dbName, measurementName);
    for (String entry : measurement.getSeriesKeys()) {
        SeriesFieldMap seriesFromKey = measurement.getSeriesFromKey(entry);
        if (seriesFromKey.get(valueFieldName) != null) {
            return seriesFromKey.get(valueFieldName).isFp();
        }
    }
    throw NOT_FOUND_EXCEPTION;
}