Examples with LevelQueue com.ctriposs.sdb.LevelQueue used on opensource projects

Example 1 with LevelQueue

use of com.ctriposs.sdb.LevelQueue in project sessdb by ppdai.

the class Level0MergerTest method testCase02.

@Test
public void testCase02() throws IOException, ClassNotFoundException {
    HashMapTable[] sourceTables = new HashMapTable[4];
    LevelQueue lq0 = new LevelQueue();
    for (int i = 0; i < 4; i++) {
        sourceTables[i] = new HashMapTable(testDir, SDB.LEVEL0, System.nanoTime() + i);
        sourceTables[i].setCompressionEnabled(true);
        lq0.addFirst(sourceTables[i]);
        assertTrue(sourceTables[i].isImmutable());
    }
    // delete
    int max = AbstractMapTable.INIT_INDEX_ITEMS_PER_TABLE;
    HashMapTable table4 = sourceTables[3];
    int start = 0;
    while (start < max) {
        table4.delete(String.valueOf(start).getBytes());
        start = start + 4;
    }
    // expiration
    HashMapTable table3 = sourceTables[2];
    start = 1;
    while (start < max) {
        table3.put(String.valueOf(start).getBytes(), String.valueOf(start).getBytes(), 200, System.currentTimeMillis());
        start = start + 4;
    }
    // expire table3
    try {
        Thread.sleep(2000);
    } catch (InterruptedException e) {
    // ignore
    }
    // time to live 60 sec
    HashMapTable table2 = sourceTables[1];
    start = 2;
    while (start < max) {
        table2.put(String.valueOf(start).getBytes(), String.valueOf(start).getBytes(), 60 * 1000, System.currentTimeMillis());
        start = start + 4;
    }
    // time to live 120 sec with items that have been updated(table 2), deleted(table 4) and expired(table 3)
    HashMapTable table1 = sourceTables[0];
    start = 0;
    while (start < max) {
        table1.put(String.valueOf(start).getBytes(), String.valueOf(start).getBytes(), 120 * 1000, System.currentTimeMillis());
        start = start + 1;
    }
    LevelQueue lq1 = new LevelQueue();
    Level0Merger.mergeSort(lq0, lq1, 4, testDir, (short) 0);
    for (int i = 0; i < 4; i++) {
        assertTrue(sourceTables[i].isImmutable());
    }
    assertTrue(lq1.size() == 1);
    MMFMapTable targetTable = (MMFMapTable) lq1.poll();
    assertTrue(targetTable.getLevel() == SDB.LEVEL1);
    assertTrue(targetTable.getAppendedSize() == max);
    // validate delete
    start = 0;
    while (start < max) {
        GetResult result = targetTable.get(String.valueOf(start).getBytes());
        assertTrue(result.isFound());
        assertTrue(result.isDeleted());
        start += 4;
    }
    // validate expiration
    start = 1;
    while (start < max) {
        GetResult result = targetTable.get(String.valueOf(start).getBytes());
        assertTrue(result.isFound());
        assertTrue(result.isExpired());
        start += 4;
    }
    // validate ttl 60s
    start = 2;
    while (start < max) {
        GetResult result = targetTable.get(String.valueOf(start).getBytes());
        assertTrue(result.isFound());
        assertTrue(result.getTimeToLive() == 60 * 1000);
        start += 4;
    }
    // validate ttl 120s
    start = 3;
    while (start < max) {
        GetResult result = targetTable.get(String.valueOf(start).getBytes());
        assertTrue(result.isFound());
        assertTrue(result.getTimeToLive() == 120 * 1000);
        start += 4;
    }
    List<String> keyList = new ArrayList<String>();
    for (long i = 0; i < max; i++) {
        keyList.add(String.valueOf(i));
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (int i = 0; i < max; i++) {
        IMapEntry mapEntry = targetTable.getMapEntry(i);
        assertTrue(mapEntry.getIndex() == i);
        assertTrue(new String(mapEntry.getKey()).equals(keyList.get(i)));
        int intKey = Integer.parseInt(new String(mapEntry.getKey()));
        // validate disk space optimization
        if (// deleted
        intKey % 4 == 0)
            assertTrue(Arrays.equals(new byte[] { 0 }, mapEntry.getValue()));
        else if (// expired
        intKey % 4 == 1)
            assertTrue(Arrays.equals(new byte[] { 0 }, mapEntry.getValue()));
        else {
            assertTrue(new String(Snappy.uncompress(mapEntry.getValue())).equals(keyList.get(i)));
        //assertTrue(new String(mapEntry.getValue()).equals(keyList.get(i)));
        }
    }
    Random random = new Random();
    for (int i = 0; i < 1024; i++) {
        long key = random.nextInt(max);
        GetResult result = targetTable.get(String.valueOf(key).getBytes());
        assertTrue(result.isFound());
    }
    targetTable.close();
    targetTable.delete();
}

Example 2 with LevelQueue

use of com.ctriposs.sdb.LevelQueue in project sessdb by ppdai.

the class Level0Merger method run.

@Override
public void run() {
    while (!stop) {
        try {
            LevelQueue levelQueue0 = levelQueueList.get(SDB.LEVEL0);
            if (levelQueue0 != null && levelQueue0.size() >= DEFAULT_MERGE_WAYS) {
                log.info("Start running level 0 merge thread at " + DateFormatter.formatCurrentDate());
                log.info("Current queue size at level 0 is " + levelQueue0.size());
                long start = System.nanoTime();
                LevelQueue levelQueue1 = levelQueueList.get(SDB.LEVEL1);
                mergeSort(levelQueue0, levelQueue1, DEFAULT_MERGE_WAYS, sdb.getDir(), shard);
                stats.recordMerging(SDB.LEVEL0, System.nanoTime() - start);
                log.info("Stopped running level 0 merge thread at " + DateFormatter.formatCurrentDate());
            } else {
                Thread.sleep(MAX_SLEEP_TIME);
            }
        } catch (Exception ex) {
            log.error("Error occured in the level0 merge dumper", ex);
        }
    }
    this.countDownLatch.countDown();
    log.info("Stopped level 0 merge thread " + this.getName());
}

Example 3 with LevelQueue

use of com.ctriposs.sdb.LevelQueue in project sessdb by ppdai.

the class Level1Merger method run.

@Override
public void run() {
    while (!stop) {
        try {
            boolean merged = false;
            LevelQueue lq1 = levelQueueList.get(SDB.LEVEL1);
            LevelQueue lq2 = levelQueueList.get(SDB.LEVEL2);
            boolean hasLevel2MapTable = lq2.size() > 0;
            if ((!hasLevel2MapTable && lq1.size() >= DEFAULT_MERGE_WAYS) || (hasLevel2MapTable && lq1.size() >= DEFAULT_MERGE_WAYS - 1)) {
                log.info("Start running level 1 merging at " + DateFormatter.formatCurrentDate());
                log.info("Current queue size at level 1 is " + lq1.size());
                log.info("Current queue size at level 2 is " + lq2.size());
                long start = System.nanoTime();
                mergeSort(lq1, lq2, DEFAULT_MERGE_WAYS, sdb.getDir(), shard);
                stats.recordMerging(SDB.LEVEL1, System.nanoTime() - start);
                merged = true;
                log.info("End running level 1 to 2 merging at " + DateFormatter.formatCurrentDate());
            }
            if (!merged) {
                Thread.sleep(MAX_SLEEP_TIME);
            }
        } catch (Exception ex) {
            log.error("Error occured in the level 1 to 2 merger", ex);
        }
    }
    this.countDownLatch.countDown();
    log.info("Stopped level 1 to 2 merge thread " + this.getName());
}

Example 4 with LevelQueue

use of com.ctriposs.sdb.LevelQueue in project sessdb by ppdai.

the class FileStatsCollector method run.

@Override
public void run() {
    while (!stop) {
        try {
            for (int level = 0; level <= SDB.MAX_LEVEL; ++level) {
                long fileSize = 0;
                int fileCount = 0;
                for (int shard = 0; shard < levelQueueLists.length; ++shard) {
                    LevelQueue queue = levelQueueLists[shard].get(level);
                    queue.getReadLock().lock();
                    try {
                        for (AbstractMapTable table : queue) {
                            fileSize += table.getBackFileSize();
                        }
                        fileCount += queue.size();
                    } finally {
                        queue.getReadLock().unlock();
                    }
                }
                stats.recordFileStats(level, fileCount, fileSize);
            }
            Thread.sleep(MAX_SLEEP_TIME);
        } catch (Exception ex) {
            log.error("Error occurred in the file stats collector", ex);
        }
    }
}

Example 5 with LevelQueue

use of com.ctriposs.sdb.LevelQueue in project sessdb by ppdai.

the class Level0MergerTest method testCase01.

@Test
public void testCase01() throws IOException, ClassNotFoundException {
    String value = TestUtil.randomString(128);
    HashMapTable[] sourceTables = new HashMapTable[4];
    LevelQueue lq0 = new LevelQueue();
    for (int i = 0; i < 4; i++) {
        sourceTables[i] = new HashMapTable(testDir, SDB.LEVEL0, System.nanoTime() + i);
        sourceTables[i].setCompressionEnabled(true);
        lq0.addFirst(sourceTables[i]);
        assertTrue(sourceTables[i].isImmutable());
    }
    int totalCount = 0;
    int max = 0;
    for (int i = 0; i < 4; i++) {
        int start = i;
        HashMapTable table = sourceTables[i];
        while (table.put(String.valueOf(start).getBytes(), value.getBytes(), AbstractMapTable.NO_TIMEOUT, System.currentTimeMillis(), false)) {
            totalCount++;
            if (start > max)
                max = start;
            start = start + 4;
        }
    }
    LevelQueue lq1 = new LevelQueue();
    long start = System.currentTimeMillis();
    Level0Merger.mergeSort(lq0, lq1, 4, testDir, (short) 1);
    long end = System.currentTimeMillis();
    System.out.println("Time spent to merge " + totalCount + " items in 4 ways  is " + (end - start) / 1000 + "s");
    for (int i = 0; i < 4; i++) {
        assertTrue(sourceTables[i].isImmutable());
    }
    assertTrue(lq1.size() == 1);
    MMFMapTable targetTable = (MMFMapTable) lq1.poll();
    assertTrue(targetTable.getLevel() == SDB.LEVEL1);
    assertTrue(targetTable.getAppendedSize() == totalCount);
    List<String> keyList = new ArrayList<String>();
    for (long i = 0; i < totalCount; i++) {
        keyList.add(String.valueOf(i));
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (int i = 0; i < totalCount; i++) {
        IMapEntry mapEntry = targetTable.getMapEntry(i);
        assertTrue(mapEntry.getIndex() == i);
        assertTrue(new String(mapEntry.getKey()).equals(keyList.get(i)));
        assertTrue(new String(Snappy.uncompress(mapEntry.getValue())).equals(value));
    }
    start = System.currentTimeMillis();
    Random random = new Random();
    for (int i = 0; i < 1024; i++) {
        long key = random.nextInt(totalCount);
        GetResult result = targetTable.get(String.valueOf(key).getBytes());
        assertTrue(result.isFound());
    }
    end = System.currentTimeMillis();
    System.out.println("Time to lookup 1024 random key in the target table is " + (end - start) + "ms");
    targetTable.close();
    targetTable.delete();
}