Examples with SeriesFieldMap com.srotya.sidewinder.core.storage.SeriesFieldMap used on opensource projects

Example 6 with SeriesFieldMap

use of com.srotya.sidewinder.core.storage.SeriesFieldMap in project sidewinder by srotya.

the class TestMappedBitmapTagIndex method testDiskTagIndexFilterEvaluation.

@Test
public void testDiskTagIndexFilterEvaluation() throws IOException, InterruptedException {
    MiscUtils.delete(new File("target/s7"));
    String indexDir = "target/s7";
    new File(indexDir).mkdirs();
    PersistentMeasurement m = new PersistentMeasurement();
    Map<String, String> conf = new HashMap<>();
    m.configure(conf, engine, "d", "m", "target/s7/i/bitmap", "target/s7/d/bitmap", new DBMetadata(), null);
    MappedBitmapTagIndex index = new MappedBitmapTagIndex(indexDir, "s7", m);
    for (int i = 0; i < 10_000; i++) {
        index.index("key", String.valueOf(i), i);
        String valueOf = String.valueOf(i);
        m.getSeriesListAsList().add(new SeriesFieldMap(valueOf));
    }
    TagFilter filter = new SimpleTagFilter(FilterType.GREATER_THAN, "key", "9");
    Set<String> keys = index.searchRowKeysForTagFilter(filter);
    assertEquals(1110, keys.size());
    filter = new SimpleTagFilter(FilterType.GREATER_THAN_EQUALS, "key", "9");
    keys = index.searchRowKeysForTagFilter(filter);
    assertEquals(1111, keys.size());
    filter = new SimpleTagFilter(FilterType.LESS_THAN, "key", "10");
    keys = index.searchRowKeysForTagFilter(filter);
    assertEquals(2, keys.size());
    filter = new SimpleTagFilter(FilterType.LESS_THAN_EQUALS, "key", "1000");
    keys = index.searchRowKeysForTagFilter(filter);
    // keys.stream().forEach(System.out::println);
    assertEquals(5, keys.size());
}

Example 7 with SeriesFieldMap

use of com.srotya.sidewinder.core.storage.SeriesFieldMap in project sidewinder by srotya.

the class DiskStorageEngine method isMeasurementFieldFP.

@Override
public boolean isMeasurementFieldFP(String dbName, String measurementName, String valueFieldName) throws IOException {
    if (!checkIfExists(dbName, measurementName)) {
        throw NOT_FOUND_EXCEPTION;
    }
    Map<String, Measurement> dbMap = getOrCreateDatabase(dbName);
    // check and create measurement map
    Measurement measurement = getOrCreateMeasurement(dbMap, measurementName, dbName);
    for (String entry : measurement.getSeriesKeys()) {
        SeriesFieldMap seriesFromKey = measurement.getSeriesFromKey(entry);
        if (seriesFromKey.get(valueFieldName) != null) {
            return seriesFromKey.get(valueFieldName).isFp();
        }
    }
    throw NOT_FOUND_EXCEPTION;
}

Example 8 with SeriesFieldMap

use of com.srotya.sidewinder.core.storage.SeriesFieldMap in project sidewinder by srotya.

the class PersistentMeasurement method loadTimeseriesFromMeasurements.

@Override
public void loadTimeseriesFromMeasurements() throws IOException {
    String mdFilePath = getMetadataPath();
    File file = new File(mdFilePath);
    if (!file.exists()) {
        logger.warning("Metadata file missing for measurement:" + measurementName);
        return;
    } else {
        logger.fine("Metadata file exists:" + file.getAbsolutePath());
    }
    List<String> seriesEntries = MiscUtils.readAllLines(file);
    try {
        loadSeriesEntries(seriesEntries);
    } catch (Exception e) {
        throw new IOException(e);
    }
    Map<String, List<Entry<String, BufferObject>>> seriesBuffers = malloc.seriesBufferMap();
    for (Entry<String, List<Entry<String, BufferObject>>> entry : seriesBuffers.entrySet()) {
        String[] split = entry.getKey().split(SERIESID_SEPARATOR);
        Integer seriesId = seriesMap.get(split[0]);
        SeriesFieldMap ts = seriesList.get(seriesId);
        List<Entry<String, BufferObject>> list = entry.getValue();
        if (list != null) {
            try {
                ts.get(split[1]).loadBucketMap(list);
            } catch (Exception e) {
                logger.log(Level.SEVERE, "Failed to load bucket map for:" + entry.getKey() + ":" + measurementName, e);
            }
        }
    }
    if (compactOnStart) {
        compact();
    }
    logger.info("Loaded measurement:" + measurementName);
}

Example 9 with SeriesFieldMap

use of com.srotya.sidewinder.core.storage.SeriesFieldMap in project sidewinder by srotya.

the class PersistentMeasurement method loadEntry.

private void loadEntry(String entry) {
    String[] split = entry.split(MD_SEPARATOR);
    String seriesId = split[0];
    logger.fine("Loading Timeseries:" + seriesId);
    try {
        String timeBucketSize = split[2];
        String isFp = split[1];
        TimeSeries timeSeries = new TimeSeries(this, compressionCodec, compactionCodec, seriesId, Integer.parseInt(timeBucketSize), metadata, Boolean.parseBoolean(isFp), conf);
        String[] split2 = seriesId.split(SERIESID_SEPARATOR);
        String valueField = split2[1];
        seriesId = split2[0];
        Integer seriesIdx = seriesMap.get(seriesId);
        SeriesFieldMap m = null;
        if (seriesIdx == null) {
            seriesIdx = Integer.parseInt(split[3], 16);
            m = new SeriesFieldMap(seriesId);
            seriesMap.put(seriesId, seriesIdx);
            seriesList.add(seriesIdx, m);
        } else {
            m = seriesList.get(seriesIdx);
        }
        m.addSeries(valueField, timeSeries);
        logger.fine("Intialized Timeseries:" + seriesId);
    } catch (NumberFormatException | IOException e) {
        logger.log(Level.SEVERE, "Failed to load series:" + entry, e);
    }
}

Example 10 with SeriesFieldMap

use of com.srotya.sidewinder.core.storage.SeriesFieldMap in project sidewinder by srotya.

the class PersistentMeasurement method getOrCreateTimeSeries.

@Override
public TimeSeries getOrCreateTimeSeries(String valueFieldName, List<String> tags, int timeBucketSize, boolean fp, Map<String, String> conf) throws IOException {
    Collections.sort(tags);
    String seriesId = constructSeriesId(tags, tagIndex);
    int index = 0;
    SeriesFieldMap seriesFieldMap = getSeriesFromKey(seriesId);
    if (seriesFieldMap == null) {
        lock.lock();
        try {
            if ((seriesFieldMap = getSeriesFromKey(seriesId)) == null) {
                index = seriesList.size();
                Measurement.indexRowKey(tagIndex, index, tags);
                seriesFieldMap = new SeriesFieldMap(seriesId);
                seriesList.add(seriesFieldMap);
                seriesMap.put(seriesId, index);
                if (enableMetricsCapture) {
                    metricsTimeSeriesCounter.inc();
                }
                logger.fine("Created new series:" + seriesId + "\t");
            } else {
                index = seriesMap.get(seriesId);
            }
        } finally {
            lock.unlock();
        }
    } else {
        index = seriesMap.get(seriesId);
    }
    TimeSeries series = seriesFieldMap.get(valueFieldName);
    if (series == null) {
        lock.lock();
        try {
            if ((series = seriesFieldMap.get(valueFieldName)) == null) {
                String seriesId2 = seriesId + SERIESID_SEPARATOR + valueFieldName;
                series = new TimeSeries(this, compressionCodec, compactionCodec, seriesId2, timeBucketSize, metadata, fp, conf);
                seriesFieldMap.addSeries(valueFieldName, series);
                appendTimeseriesToMeasurementMetadata(seriesId2, fp, timeBucketSize, index);
                logger.fine("Created new timeseries:" + seriesFieldMap + " for measurement:" + measurementName + "\t" + seriesId + "\t" + metadata.getRetentionHours() + "\t" + seriesList.size());
            }
        } finally {
            lock.unlock();
        }
    }
    return series;
}