Search in sources :

Example 6 with ByteString

use of com.srotya.sidewinder.core.storage.ByteString in project sidewinder by srotya.

the class PersistentMeasurement method loadTimeseriesInMeasurements.

@Override
public void loadTimeseriesInMeasurements() throws IOException {
    String fieldFilePath = getFieldMetadataPath();
    File file = new File(fieldFilePath);
    if (!file.exists()) {
        logger.warning("Field file missing for measurement:" + measurementName);
        return;
    } else {
        logger.fine("Field file exists:" + file.getAbsolutePath());
    }
    List<String> fieldEntries = MiscUtils.readAllLines(file);
    loadFieldList(fieldEntries);
    String mdFilePath = getMetadataPath();
    file = new File(mdFilePath);
    if (!file.exists()) {
        logger.warning("Metadata file missing for measurement:" + measurementName);
        return;
    } else {
        logger.fine("Metadata file exists:" + file.getAbsolutePath());
    }
    List<String> seriesEntries = MiscUtils.readAllLines(file);
    try {
        loadSeriesEntries(seriesEntries);
    } catch (Exception e) {
        e.printStackTrace();
        throw new IOException(e);
    }
    ByteStringCache localCache = ByteStringCache.instance();
    Map<ByteString, List<Entry<Integer, BufferObject>>> seriesBuffers = malloc.seriesBufferMap();
    for (Entry<ByteString, List<Entry<Integer, BufferObject>>> entry : seriesBuffers.entrySet()) {
        ByteString[] split = entry.getKey().split(SERIESID_SEPARATOR);
        ByteString seriesId = localCache.get(split[0]);
        Integer seriesIndex = seriesMap.get(seriesId);
        Series series = seriesList.get(seriesIndex);
        if (series.getSeriesId() != seriesId) {
            seriesMap.put(seriesId, seriesIndex);
            series.setSeriesId(seriesId);
        }
        List<Entry<Integer, BufferObject>> list = entry.getValue();
        if (list != null) {
            try {
                String fieldName = split[1].toString();
                series.loadBuffers(this, fieldName, list, this.getConf());
            } catch (Exception e) {
                logger.log(Level.SEVERE, "Failed to load bucket map for:" + entry.getKey() + ":" + measurementName, e);
            }
        }
    }
    if (compactOnStart) {
        compact();
    }
    logger.info("Loaded measurement:" + measurementName);
}
Also used : ByteStringCache(com.srotya.sidewinder.core.storage.ByteString.ByteStringCache) BufferObject(com.srotya.sidewinder.core.storage.BufferObject) ByteString(com.srotya.sidewinder.core.storage.ByteString) ByteString(com.srotya.sidewinder.core.storage.ByteString) IOException(java.io.IOException) IOException(java.io.IOException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Series(com.srotya.sidewinder.core.storage.Series) Entry(java.util.Map.Entry) ArrayList(java.util.ArrayList) List(java.util.List) File(java.io.File)

Example 7 with ByteString

use of com.srotya.sidewinder.core.storage.ByteString in project sidewinder by srotya.

the class TestDownSamplingIFunction method testBasicSumDownSampling.

@Test
public void testBasicSumDownSampling() throws IOException {
    Field tField = new TimeField(measurement, new LinkedByteString().concat(new ByteString("time")), 121213, new HashMap<>());
    long ts = 1546755994280L;
    for (int i = 0; i < 100; i++) {
        tField.addDataPoint(measurement, ts + i * 1000);
    }
    Field vField = new ValueField(measurement, new LinkedByteString().concat(new ByteString("field1")), 121213, new HashMap<>());
    for (int i = 0; i < 100; i++) {
        vField.addDataPoint(measurement, i);
    }
    DataPointIterator itr = new DataPointIterator(tField.queryReader(null, new NoLock()), vField.queryReader(null, new NoLock()));
    DownsampleFunction f = new DownsampleFunction(itr, 5, TimeUnit.SECONDS, ((x, y) -> (x + y)));
    int c = 0;
    while (f.hasNext()) {
        DataPoint next = f.next();
        if (c < 20) {
            assertEquals(ts + c * 1000 * 5, next.getTimestamp());
        }
        c++;
    }
    assertEquals(21, c);
    itr = new DataPointIterator(tField.queryReader(null, new NoLock()), vField.queryReader(null, new NoLock()));
    f = new DownsampleFunction(itr, 10, TimeUnit.SECONDS, ((x, y) -> (x + y) / 2));
    c = 0;
    while (f.hasNext()) {
        DataPoint next = f.next();
        if (c < 10) {
            assertEquals(ts + c * 1000 * 10, next.getTimestamp());
        }
        c++;
    }
    assertEquals(10, c, 1);
}
Also used : DownsampleFunction(com.srotya.sidewinder.core.functions.iterative.DownsampleFunction) NoLock(com.srotya.sidewinder.core.storage.NoLock) IOException(java.io.IOException) HashMap(java.util.HashMap) Test(org.junit.Test) DataPoint(com.srotya.sidewinder.core.storage.DataPoint) LinkedByteString(com.srotya.sidewinder.core.storage.LinkedByteString) TimeUnit(java.util.concurrent.TimeUnit) Field(com.srotya.sidewinder.core.storage.Field) ValueField(com.srotya.sidewinder.core.storage.ValueField) CompressionFactory(com.srotya.sidewinder.core.storage.compression.CompressionFactory) ByteString(com.srotya.sidewinder.core.storage.ByteString) MockMeasurement(com.srotya.sidewinder.core.storage.MockMeasurement) Assert.assertEquals(org.junit.Assert.assertEquals) DataPointIterator(com.srotya.sidewinder.core.storage.DataPointIterator) TimeField(com.srotya.sidewinder.core.storage.TimeField) Before(org.junit.Before) LinkedByteString(com.srotya.sidewinder.core.storage.LinkedByteString) ByteString(com.srotya.sidewinder.core.storage.ByteString) DataPoint(com.srotya.sidewinder.core.storage.DataPoint) Field(com.srotya.sidewinder.core.storage.Field) ValueField(com.srotya.sidewinder.core.storage.ValueField) TimeField(com.srotya.sidewinder.core.storage.TimeField) TimeField(com.srotya.sidewinder.core.storage.TimeField) DataPointIterator(com.srotya.sidewinder.core.storage.DataPointIterator) DataPoint(com.srotya.sidewinder.core.storage.DataPoint) NoLock(com.srotya.sidewinder.core.storage.NoLock) LinkedByteString(com.srotya.sidewinder.core.storage.LinkedByteString) ValueField(com.srotya.sidewinder.core.storage.ValueField) DownsampleFunction(com.srotya.sidewinder.core.functions.iterative.DownsampleFunction) Test(org.junit.Test)

Example 8 with ByteString

use of com.srotya.sidewinder.core.storage.ByteString in project sidewinder by srotya.

the class MappedSetTagIndex method searchRowKeysForTagFilter.

@Override
public Set<ByteString> searchRowKeysForTagFilter(TagFilter tagFilterTree) {
    Set<ByteString> hexKeys = TagIndex.stringSetToByteSet(evalFilterForTags(tagFilterTree), new HashSet<>());
    Set<ByteString> rowKeys = new HashSet<>();
    List<Series> list = m.getSeriesList();
    for (ByteString val : hexKeys) {
        ByteString[] split = val.split(SEPERATOR);
        rowKeys.add(new ByteString(list.get(Integer.parseInt(split[split.length - 1].toString(), 16)).getSeriesId().toString()));
    }
    return rowKeys;
}
Also used : Series(com.srotya.sidewinder.core.storage.Series) ByteString(com.srotya.sidewinder.core.storage.ByteString) HashSet(java.util.HashSet)

Example 9 with ByteString

use of com.srotya.sidewinder.core.storage.ByteString in project sidewinder by srotya.

the class PersistentMeasurement method appendTimeseriesToMeasurementMetadata.

@Override
public synchronized void appendTimeseriesToMeasurementMetadata(ByteString fieldId, int idx) throws IOException {
    String line = fieldId.toString() + MD_SEPARATOR + Integer.toHexString(idx);
    DiskStorageEngine.appendLineToFile(line, prMetadata);
}
Also used : ByteString(com.srotya.sidewinder.core.storage.ByteString)

Example 10 with ByteString

use of com.srotya.sidewinder.core.storage.ByteString in project sidewinder by srotya.

the class TestBasicSingleFunctions method before.

@Before
public void before() throws IOException {
    measurement = new MockMeasurement(32768, 100);
    TimeField.compressionClass = CompressionFactory.getTimeClassByName("byzantine");
    Field tField = new TimeField(measurement, new LinkedByteString().concat(new ByteString("time")), 121213, new HashMap<>());
    long ts = 1546755991280L;
    for (int i = 0; i < 200; i++) {
        tField.addDataPoint(measurement, ts + i * 1000);
    }
    Field vField = new ValueField(measurement, new LinkedByteString().concat(new ByteString("field1")), 121213, new HashMap<>());
    for (int i = 0; i < 100; i++) {
        vField.addDataPoint(measurement, i * 1L);
    }
    for (int i = 100; i > 0; i--) {
        vField.addDataPoint(measurement, i * 1L);
    }
    itr = new DataPointIterator(tField.queryReader(null, new NoLock()), vField.queryReader(null, new NoLock()));
}
Also used : Field(com.srotya.sidewinder.core.storage.Field) ValueField(com.srotya.sidewinder.core.storage.ValueField) TimeField(com.srotya.sidewinder.core.storage.TimeField) TimeField(com.srotya.sidewinder.core.storage.TimeField) DataPointIterator(com.srotya.sidewinder.core.storage.DataPointIterator) MockMeasurement(com.srotya.sidewinder.core.storage.MockMeasurement) LinkedByteString(com.srotya.sidewinder.core.storage.LinkedByteString) ByteString(com.srotya.sidewinder.core.storage.ByteString) NoLock(com.srotya.sidewinder.core.storage.NoLock) LinkedByteString(com.srotya.sidewinder.core.storage.LinkedByteString) DataPoint(com.srotya.sidewinder.core.storage.DataPoint) ValueField(com.srotya.sidewinder.core.storage.ValueField) Before(org.junit.Before)

Aggregations

ByteString (com.srotya.sidewinder.core.storage.ByteString)19 LinkedByteString (com.srotya.sidewinder.core.storage.LinkedByteString)9 Series (com.srotya.sidewinder.core.storage.Series)7 Test (org.junit.Test)6 DataPointIterator (com.srotya.sidewinder.core.storage.DataPointIterator)5 Field (com.srotya.sidewinder.core.storage.Field)5 NoLock (com.srotya.sidewinder.core.storage.NoLock)5 TimeField (com.srotya.sidewinder.core.storage.TimeField)5 ValueField (com.srotya.sidewinder.core.storage.ValueField)5 HashMap (java.util.HashMap)5 BufferObject (com.srotya.sidewinder.core.storage.BufferObject)4 File (java.io.File)4 IOException (java.io.IOException)4 DataPoint (com.srotya.sidewinder.core.storage.DataPoint)3 MockMeasurement (com.srotya.sidewinder.core.storage.MockMeasurement)3 MappedByteBuffer (java.nio.MappedByteBuffer)3 HashSet (java.util.HashSet)3 Before (org.junit.Before)3 SimpleTagFilter (com.srotya.sidewinder.core.filters.SimpleTagFilter)2 DownsampleFunction (com.srotya.sidewinder.core.functions.iterative.DownsampleFunction)2