Examples with ByteString com.srotya.sidewinder.core.storage.ByteString used on opensource projects

Example 1 with ByteString

use of com.srotya.sidewinder.core.storage.ByteString in project sidewinder by srotya.

the class Archiver method deserializeFromStream.

public static TimeSeriesArchivalObject deserializeFromStream(DataInputStream bis) throws IOException {
    TimeSeriesArchivalObject bucketWraper = new TimeSeriesArchivalObject();
    bucketWraper.setDb(bis.readUTF());
    bucketWraper.setMeasurement(bis.readUTF());
    bucketWraper.setSeriesKey(new ByteString(bis.readUTF()));
    bucketWraper.setTsBucket(bis.readInt());
    byte[] buf = new byte[bis.readInt()];
    bis.read(buf);
    bucketWraper.setData(buf);
    return bucketWraper;
}

Example 2 with ByteString

use of com.srotya.sidewinder.core.storage.ByteString in project sidewinder by srotya.

the class TestMappedBitmapTagIndex method testIndexRecovery.

@Test
public void testIndexRecovery() throws IOException, InterruptedException {
    MiscUtils.delete(new File("target/s9"));
    String indexDir = "target/s9";
    new File(indexDir).mkdirs();
    PersistentMeasurement m = new PersistentMeasurement();
    Map<String, String> conf = new HashMap<>();
    m.configure(conf, engine, 4096, "d", "m", "target/s9/i/bitmap", "target/s9/d/bitmap", new DBMetadata(), null);
    MappedBitmapTagIndex index = new MappedBitmapTagIndex();
    index.configure(conf, "target/s9/i/bitmap", m);
    long ts = System.currentTimeMillis();
    for (int i = 0; i < 10_000; i++) {
        index.index("key", String.valueOf(i), i);
        ByteString valueOf = new ByteString(String.valueOf(i));
        m.getSeriesListAsList().add(new Series(valueOf, i));
    }
    ts = System.currentTimeMillis() - ts;
    System.out.println("Time:" + ts);
    for (int i = 0; i < 10_000; i++) {
        assertEquals(new HashSet<>(Arrays.asList(new ByteString(String.valueOf(i)))), index.searchRowKeysForTagFilter(new SimpleTagFilter(FilterType.EQUALS, "key", String.valueOf(i))));
    }
    for (int k = 0; k < 10; k++) {
        index = new MappedBitmapTagIndex();
        index.configure(conf, "target/s9/i/bitmap", m);
        for (int i = 0; i < 10_000; i++) {
            assertEquals(new HashSet<>(Arrays.asList(new ByteString(String.valueOf(i)))), index.searchRowKeysForTagFilter(new SimpleTagFilter(FilterType.EQUALS, "key", String.valueOf(i))));
        }
    }
}

Example 3 with ByteString

use of com.srotya.sidewinder.core.storage.ByteString in project sidewinder by srotya.

the class TestMappedBitmapTagIndex method testMultiIndexRecovery.

@Test
public void testMultiIndexRecovery() throws IOException, InterruptedException {
    MiscUtils.delete(new File("target/s9"));
    String indexDir = "target/s9";
    new File(indexDir).mkdirs();
    PersistentMeasurement m = new PersistentMeasurement();
    Map<String, String> conf = new HashMap<>();
    m.configure(conf, engine, 4096, "d", "m", "target/s9/i/bitmap", "target/s9/d/bitmap", new DBMetadata(), null);
    MappedBitmapTagIndex index = new MappedBitmapTagIndex();
    index.configure(conf, "target/s9/i/bitmap", m);
    long ts = System.currentTimeMillis();
    for (int i = 0; i < 10_000; i++) {
        index.index("key", String.valueOf(i), i);
        index.index("key", String.valueOf(i - 1), i);
        ByteString valueOf = new ByteString(String.valueOf(i));
        m.getSeriesListAsList().add(new Series(valueOf, i));
    }
    ts = System.currentTimeMillis() - ts;
    System.out.println("Time:" + ts);
    for (int i = 0; i < 9999; i++) {
        assertEquals(new HashSet<>(Arrays.asList(new ByteString(String.valueOf(i)), new ByteString(String.valueOf(i + 1)))), index.searchRowKeysForTagFilter(new SimpleTagFilter(FilterType.EQUALS, "key", String.valueOf(i))));
    }
    for (int k = 0; k < 10; k++) {
        index = new MappedBitmapTagIndex();
        index.configure(conf, "target/s9/i/bitmap", m);
        for (int i = 1; i < 9999; i++) {
            assertEquals(new HashSet<>(Arrays.asList(new ByteString(String.valueOf(i)), new ByteString(String.valueOf(i + 1)))), index.searchRowKeysForTagFilter(new SimpleTagFilter(FilterType.EQUALS, "key", String.valueOf(i))));
        }
    }
}

Example 4 with ByteString

use of com.srotya.sidewinder.core.storage.ByteString in project sidewinder by srotya.

the class DiskMalloc method seriesBufferMap.

@Override
public Map<ByteString, List<Entry<Integer, BufferObject>>> seriesBufferMap() throws FileNotFoundException, IOException {
    Map<String, MappedByteBuffer> bufferMap = new ConcurrentHashMap<>();
    File[] listFiles = new File(dataDirectory).listFiles(new FilenameFilter() {

        @Override
        public boolean accept(File dir, String name) {
            return name.endsWith(".dat");
        }
    });
    Arrays.sort(listFiles, new Comparator<File>() {

        @Override
        public int compare(File o1, File o2) {
            return o1.getName().compareTo(o2.getName());
        }
    });
    for (File dataFile : listFiles) {
        try {
            RandomAccessFile raf = new RandomAccessFile(dataFile, "rw");
            MappedByteBuffer map = raf.getChannel().map(MapMode.READ_WRITE, 0, dataFile.length());
            bufferMap.put(dataFile.getName(), map);
            logger.info("Recovering data file:" + dataDirectory + "/" + dataFile.getName());
            raf.close();
        } catch (Exception e) {
            logger.log(Level.SEVERE, "Failed to recover data files for measurement:" + measurementName, e);
        }
    }
    // files and this will prevent them from being overwritten
    if (listFiles.length > 0) {
        fcnt = Integer.parseInt(listFiles[listFiles.length - 1].getName().replace("data-", "").replace(".dat", "")) + 1;
    }
    Map<ByteString, List<Entry<Integer, BufferObject>>> seriesBuffers = new HashMap<>();
    sliceMappedBuffersForBuckets(bufferMap, seriesBuffers);
    return seriesBuffers;
}

Example 5 with ByteString

use of com.srotya.sidewinder.core.storage.ByteString in project sidewinder by srotya.

the class DiskMalloc method createNewBuffer.

@Override
public BufferObject createNewBuffer(LinkedByteString fieldId, Integer tsBucket, int newSize) throws IOException {
    logger.fine(() -> "Seriesid:" + fieldId + " requesting buffer of size:" + newSize);
    if (rafActiveFile == null) {
        lock.lock();
        if (rafActiveFile == null) {
            filename = new ByteString(dataDirectory + "/data-" + String.format("%012d", fcnt) + ".dat");
            rafActiveFile = new RandomAccessFile(filename.toString(), "rwd");
            offset = 0;
            logger.info("Creating new datafile for measurement:" + filename);
            memoryMappedBuffer = rafActiveFile.getChannel().map(MapMode.READ_WRITE, 0, fileMapIncrement);
            fcnt++;
            if (enableMetricsCapture) {
                metricsFileRotation.inc();
            }
        }
        lock.unlock();
    }
    lock.lock();
    try {
        if (curr + newSize < 0 || curr + newSize > memoryMappedBuffer.remaining() + 1) {
            curr = 0;
            itr++;
            offset = (((long) (fileMapIncrement)) * itr);
            // requested
            if (offset >= maxFileSize) {
                itr = 0;
                logger.info("Rotating datafile for measurement:" + measurementName + " closing active file:" + filename);
                memoryMappedBuffer.force();
                rafActiveFile.close();
                rafActiveFile = null;
                return createNewBuffer(fieldId, tsBucket, newSize);
            }
            // used for GC testing and debugging
            if (oldBufferReferences != null) {
                oldBufferReferences.put(filename, new WeakReference<MappedByteBuffer>(memoryMappedBuffer));
            }
            memoryMappedBuffer = rafActiveFile.getChannel().map(MapMode.READ_WRITE, offset, fileMapIncrement);
            logger.fine("Buffer expansion:" + offset + "\t\t" + curr);
            if (enableMetricsCapture) {
                metricsBufferResize.inc();
                metricsBufferSize.inc(fileMapIncrement);
            }
        }
        LinkedByteString ptrKey = appendBufferPointersToDisk(fieldId, filename, curr, offset, newSize, tsBucket);
        ByteBuffer buf = memoryMappedBuffer.slice();
        buf.limit(newSize);
        curr = curr + newSize;
        memoryMappedBuffer.position(curr);
        logger.fine(() -> "Position:" + buf.position() + "\t" + buf.limit() + "\t" + buf.capacity());
        if (enableMetricsCapture) {
            metricsBufferCounter.inc();
        }
        return new BufferObject(ptrKey, buf);
    } finally {
        lock.unlock();
    }
}