Examples with FCMapTable com.ctriposs.sdb.table.FCMapTable used on opensource projects

Example 1 with FCMapTable

use of com.ctriposs.sdb.table.FCMapTable in project sessdb by ppdai.

the class FCMapTableTest method testEmtpy.

@Test
public void testEmtpy() throws IOException, ClassNotFoundException {
    long createdTime = System.nanoTime();
    mapTable = new FCMapTable(testDir, 1, createdTime, 1000);
    assertTrue(mapTable.getLevel() == 1);
    assertTrue(mapTable.getCreatedTime() == createdTime);
    assertTrue(mapTable.getAppendedSize() == 0);
    assertTrue(mapTable.isEmpty());
    assertTrue(mapTable.getBackFileSize() == HashMapTable.INIT_INDEX_FILE_SIZE + HashMapTable.INIT_DATA_FILE_SIZE + HashMapTable.INDEX_ITEM_LENGTH);
    assertFalse(mapTable.isUsable());
    try {
        @SuppressWarnings("unused") GetResult result = mapTable.get("empty".getBytes());
        fail();
    } catch (IllegalArgumentException iae) {
    }
    try {
        mapTable.getMapEntry(-1);
        fail();
    } catch (IllegalArgumentException iae) {
    }
    try {
        mapTable.getMapEntry(0);
    } catch (IllegalArgumentException iae) {
    }
}

Example 2 with FCMapTable

use of com.ctriposs.sdb.table.FCMapTable in project sessdb by ppdai.

the class FCMapTableTest method testLoopAndReopen.

@Test
public void testLoopAndReopen() throws IOException, ClassNotFoundException {
    long createdTime = System.nanoTime();
    int loop = 32 * 1024;
    mapTable = new FCMapTable(testDir, 1, createdTime, loop);
    assertTrue(mapTable.getLevel() == 1);
    assertTrue(mapTable.getCreatedTime() == createdTime);
    assertTrue(mapTable.getAppendedSize() == 0);
    assertTrue(mapTable.isEmpty());
    assertTrue(mapTable.getBackFileSize() == HashMapTable.INIT_INDEX_FILE_SIZE + HashMapTable.INIT_DATA_FILE_SIZE + HashMapTable.INDEX_ITEM_LENGTH);
    assertFalse(mapTable.isUsable());
    List<byte[]> list = new ArrayList<byte[]>();
    for (int i = 0; i < loop; i++) {
        list.add(("key" + i).getBytes());
    }
    Collections.sort(list, new Comparator<byte[]>() {

        @Override
        public int compare(byte[] arg0, byte[] arg1) {
            int hash0 = Arrays.hashCode(arg0);
            int hash1 = Arrays.hashCode(arg1);
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (int i = 0; i < loop; i++) {
        mapTable.appendNew(list.get(i), ("value" + i).getBytes(), -1);
    }
    assertTrue(mapTable.getAppendedSize() == loop);
    long start = System.currentTimeMillis();
    for (int i = 0; i < loop; i++) {
        GetResult result = mapTable.get(("key" + i).getBytes());
        assertTrue(result.isFound());
    }
    long time = System.currentTimeMillis() - start;
    System.out.printf("Get %,d K ops per second%n", (int) (loop / time));
    GetResult result = mapTable.get(("key" + loop).getBytes());
    assertFalse(result.isFound());
    assertFalse(result.isDeleted() || result.isExpired());
    mapTable.markUsable(true);
    mapTable.saveMetadata();
    mapTable.close();
    mapTable = new FCMapTable(testDir, mapTable.getFileName());
    assertTrue(mapTable.isUsable());
    assertTrue(mapTable.getAppendedSize() == loop);
    for (int i = 0; i < loop; i++) {
        result = mapTable.get(("key" + i).getBytes());
        assertTrue(result.isFound());
    }
    result = mapTable.get(("key" + loop).getBytes());
    assertFalse(result.isFound());
    assertFalse(result.isDeleted() || result.isExpired());
    mapTable.markUsable(false);
    mapTable.saveMetadata();
    mapTable.close();
    mapTable = new FCMapTable(testDir, mapTable.getFileName());
    assertFalse(mapTable.isUsable());
    assertTrue(mapTable.getAppendedSize() == loop);
    for (int i = 0; i < loop; i++) {
        result = mapTable.get(("key" + i).getBytes());
        assertTrue(result.isFound());
    }
}

Example 3 with FCMapTable

use of com.ctriposs.sdb.table.FCMapTable in project sessdb by ppdai.

the class Level1Merger method mergeSort.

public static void mergeSort(LevelQueue lq1, LevelQueue lq2, int ways, String dir, short shard) throws IOException, ClassNotFoundException {
    boolean hasLevel2MapTable = lq2.size() > 0;
    List<AbstractMapTable> tables = new ArrayList<AbstractMapTable>(ways);
    lq1.getReadLock().lock();
    try {
        Iterator<AbstractMapTable> iter = lq1.descendingIterator();
        for (int i = 0; i < ways - 1; i++) {
            tables.add(iter.next());
        }
        if (hasLevel2MapTable) {
            tables.add(lq2.get(0));
        } else {
            tables.add(iter.next());
        }
    } finally {
        lq1.getReadLock().unlock();
    }
    long expectedInsertions = 0;
    for (AbstractMapTable table : tables) {
        expectedInsertions += table.getAppendedSize();
    }
    if (expectedInsertions > Integer.MAX_VALUE)
        expectedInsertions = Integer.MAX_VALUE;
    // target table
    AbstractSortedMapTable sortedMapTable = new FCMapTable(dir, shard, SDB.LEVEL2, System.nanoTime(), (int) expectedInsertions);
    PriorityQueue<QueueElement> pq = new PriorityQueue<QueueElement>();
    // build initial heap
    for (AbstractMapTable table : tables) {
        QueueElement qe = new QueueElement();
        qe.sortedMapTable = table;
        qe.size = qe.sortedMapTable.getAppendedSize();
        qe.index = 0;
        qe.queue = new LinkedList<IMapEntry>();
        IMapEntry me = qe.getNextMapEntry();
        if (me != null) {
            qe.key = me.getKey();
            qe.mapEntry = me;
            qe.keyHash = me.getKeyHash();
            pq.add(qe);
        }
    }
    LinkedList<IMapEntry> targetCacheQueue = new LinkedList<IMapEntry>();
    // merge sort
    while (pq.size() > 0) {
        QueueElement qe1 = pq.poll();
        // remove old/stale entries
        while (pq.peek() != null && qe1.keyHash == pq.peek().keyHash && BytesUtil.compare(qe1.key, pq.peek().key) == 0) {
            QueueElement qe2 = pq.poll();
            IMapEntry me = qe2.getNextMapEntry();
            if (me != null) {
                qe2.key = me.getKey();
                qe2.mapEntry = me;
                qe2.keyHash = me.getKeyHash();
                pq.add(qe2);
            }
        }
        // remove deleted or expired entries in final merge sorting
        if (!qe1.mapEntry.isDeleted() && !qe1.mapEntry.isExpired()) {
            targetCacheQueue.add(qe1.mapEntry);
        }
        if (targetCacheQueue.size() >= CACHED_MAP_ENTRIES * DEFAULT_MERGE_WAYS) {
            while (targetCacheQueue.size() > 0) {
                IMapEntry mapEntry = targetCacheQueue.poll();
                byte[] value = mapEntry.getValue();
                // disk space optimization
                if (mapEntry.isExpired()) {
                    continue;
                }
                sortedMapTable.appendNew(mapEntry.getKey(), mapEntry.getKeyHash(), value, mapEntry.getTimeToLive(), mapEntry.getCreatedTime(), mapEntry.isDeleted(), mapEntry.isCompressed());
            }
        }
        IMapEntry me = qe1.getNextMapEntry();
        if (me != null) {
            qe1.key = me.getKey();
            qe1.mapEntry = me;
            qe1.keyHash = me.getKeyHash();
            pq.add(qe1);
        }
    }
    // remaining cached entries
    while (targetCacheQueue.size() > 0) {
        IMapEntry mapEntry = targetCacheQueue.poll();
        byte[] value = mapEntry.getValue();
        // disk space optimization
        if (mapEntry.isExpired()) {
            continue;
        }
        sortedMapTable.appendNew(mapEntry.getKey(), mapEntry.getKeyHash(), value, mapEntry.getTimeToLive(), mapEntry.getCreatedTime(), mapEntry.isDeleted(), mapEntry.isCompressed());
    }
    // persist metadata
    sortedMapTable.reMap();
    sortedMapTable.saveMetadata();
    // switching
    lq1.getWriteLock().lock();
    lq2.getWriteLock().lock();
    try {
        for (int i = 0; i < ways - 1; i++) {
            lq1.removeLast();
        }
        if (hasLevel2MapTable) {
            lq2.removeLast();
        } else {
            lq1.removeLast();
        }
        for (AbstractMapTable table : tables) {
            table.markUsable(false);
        }
        sortedMapTable.markUsable(true);
        lq2.addFirst(sortedMapTable);
    } finally {
        lq2.getWriteLock().unlock();
        lq1.getWriteLock().unlock();
    }
    for (AbstractMapTable table : tables) {
        table.close();
        table.delete();
    }
}

Example 4 with FCMapTable

use of com.ctriposs.sdb.table.FCMapTable in project sessdb by ppdai.

the class SDB method loadMapTables.

private void loadMapTables() throws IOException, ClassNotFoundException {
    File dirFile = new File(dir);
    if (!dirFile.exists()) {
        dirFile.mkdirs();
    }
    String[] fileNames = dirFile.list(new FilenameFilter() {

        @Override
        public boolean accept(File dir, String filename) {
            if (filename.endsWith(AbstractMapTable.INDEX_FILE_SUFFIX))
                return true;
            return false;
        }
    });
    // new DB, setup new active map table
    if (fileNames == null || fileNames.length == 0) {
        for (short i = 0; i < this.config.getShardNumber(); i++) {
            this.activeInMemTables[i] = new HashMapTable(dir, i, LEVEL0, System.nanoTime());
            this.activeInMemTables[i].markUsable(true);
            // mutable
            this.activeInMemTables[i].markImmutable(false);
            this.activeInMemTables[i].setCompressionEnabled(this.config.isCompressionEnabled());
        }
        return;
    }
    PriorityQueue<AbstractMapTable> pq = new PriorityQueue<AbstractMapTable>();
    for (String fileName : fileNames) {
        int dotIndex = fileName.lastIndexOf(".");
        if (dotIndex > 0) {
            fileName = fileName.substring(0, dotIndex);
        }
        String[] parts = fileName.split("-");
        Preconditions.checkArgument(parts != null && parts.length == 3, "on-disk table file names corrupted!");
        int level = Integer.parseInt(parts[1]);
        if (level == LEVEL0) {
            pq.add(new HashMapTable(dir, fileName));
        } else if (level == LEVEL1) {
            pq.add(new MMFMapTable(dir, fileName));
        } else {
            pq.add(new FCMapTable(dir, fileName));
        }
    }
    Preconditions.checkArgument(pq.size() > 0, "on-disk table file names corrupted!");
    // setup active map table
    for (int i = 0; i < this.config.getShardNumber(); i++) {
        AbstractMapTable table = pq.poll();
        Preconditions.checkArgument(table.getLevel() == 0, "on-disk table file names corrupted, no level 0 map tables");
        this.activeInMemTables[table.getShard()] = (HashMapTable) table;
        this.activeInMemTables[table.getShard()].markUsable(true);
        // mutable
        this.activeInMemTables[table.getShard()].markImmutable(false);
        this.activeInMemTables[table.getShard()].setCompressionEnabled(this.config.isCompressionEnabled());
    }
    while (!pq.isEmpty()) {
        AbstractMapTable table = pq.poll();
        if (table.isUsable()) {
            int level = table.getLevel();
            LevelQueue lq = levelQueueLists[table.getShard()].get(level);
            lq.addLast(table);
        } else {
            // garbage
            table.close();
            table.delete();
        }
    }
}

Example 5 with FCMapTable

use of com.ctriposs.sdb.table.FCMapTable in project sessdb by ppdai.

the class Level1MergerTest method testCase02.

@Test
public void testCase02() throws IOException, ClassNotFoundException {
    int maxSize = AbstractMapTable.INIT_INDEX_ITEMS_PER_TABLE * 4 * 4;
    //int maxSize = 1024;
    MMFMapTable[] sourceTables = new MMFMapTable[3];
    LevelQueue lq1 = new LevelQueue();
    for (int i = 0; i < 3; i++) {
        sourceTables[i] = new MMFMapTable(testDir, SDB.LEVEL1, System.nanoTime() + i, maxSize / 4, 4);
        lq1.addFirst(sourceTables[i]);
    }
    // delete
    MMFMapTable table3 = sourceTables[2];
    int start = 0;
    List<String> keyList = new ArrayList<String>();
    while (start < maxSize) {
        keyList.add(String.valueOf(start));
        start = start + 4;
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (String key : keyList) {
        table3.appendNew(key.getBytes(), Arrays.hashCode(key.getBytes()), key.getBytes(), AbstractMapTable.NO_TIMEOUT, System.currentTimeMillis(), true, false);
    }
    // expiration
    MMFMapTable table2 = sourceTables[1];
    start = 1;
    keyList = new ArrayList<String>();
    while (start < maxSize) {
        keyList.add(String.valueOf(start));
        start = start + 4;
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (String key : keyList) {
        table2.appendNew(key.getBytes(), Arrays.hashCode(key.getBytes()), key.getBytes(), 200, System.currentTimeMillis(), false, false);
    }
    // expire table2
    try {
        Thread.sleep(2000);
    } catch (InterruptedException e) {
    // ignore
    }
    // time to live 60 sec
    MMFMapTable table1 = sourceTables[0];
    start = 2;
    keyList = new ArrayList<String>();
    while (start < maxSize) {
        keyList.add(String.valueOf(start));
        start = start + 4;
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (String key : keyList) {
        table1.appendNew(key.getBytes(), Arrays.hashCode(key.getBytes()), key.getBytes(), 600 * 1000, System.currentTimeMillis(), false, false);
    }
    //int expectedInserts = (int)(table1.getAppendedSize() + table2.getAppendedSize() + table3.getAppendedSize());
    FCMapTable table4 = new FCMapTable(testDir, SDB.LEVEL2, System.nanoTime() + 3, maxSize);
    start = 0;
    keyList = new ArrayList<String>();
    while (start < maxSize) {
        keyList.add(String.valueOf(start));
        start = start + 1;
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (String key : keyList) {
        table4.appendNew(key.getBytes(), Arrays.hashCode(key.getBytes()), key.getBytes(), 1200 * 1000, System.currentTimeMillis(), false, false);
    }
    LevelQueue lq2 = new LevelQueue();
    lq2.add(table4);
    Level1Merger.mergeSort(lq1, lq2, 4, testDir, (short) 2);
    assertTrue(lq1.size() == 0);
    assertTrue(lq2.size() == 1);
    FCMapTable targetTable = (FCMapTable) lq2.poll();
    System.out.println(targetTable.getAppendedSize() + "==" + maxSize / 2);
    assertTrue(targetTable.getAppendedSize() == maxSize / 2);
    /*
		// validate delete
		start = 0;
		while(start < maxSize) {
			GetResult result = targetTable.get(String.valueOf(start).getBytes());
			assertFalse(result.isFound());
			start += 4;
		}
		
		// validate expiration
		start = 1;
		while(start < maxSize) {
			GetResult result = targetTable.get(String.valueOf(start).getBytes());
			assertFalse(result.isFound());
			start += 4;
		}
		
		// validate ttl 60s
		start = 2;
		while(start < maxSize) {
			GetResult result = targetTable.get(String.valueOf(start).getBytes());
			assertTrue(result.isFound());
			assertTrue(result.getTimeToLive() == 600 * 1000);
			start += 4;
		}
		
		// validate ttl 120s
		start = 3;
		while(start < maxSize) {
			GetResult result = targetTable.get(String.valueOf(start).getBytes());
			assertTrue(result.isFound());
			assertTrue(result.getTimeToLive() == 1200 * 1000);
			start += 4;
		}*/
    keyList = new ArrayList<String>();
    for (long i = 0; i < maxSize; i++) {
        if (i % 4 == 0 || i % 4 == 1)
            continue;
        keyList.add(String.valueOf(i));
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    int index = 0;
    for (int i = 0; i < maxSize; i++) {
        // ignore deleted & expired
        if (i % 4 == 0 || i % 4 == 1)
            continue;
        IMapEntry mapEntry = targetTable.getMapEntry(index);
        assertTrue(mapEntry.getIndex() == index);
        assertTrue(new String(mapEntry.getKey()).equals(keyList.get(index)));
        assertTrue(new String(mapEntry.getValue()).equals(keyList.get(index)));
        index++;
    }
    Random random = new Random();
    for (int i = 0; i < 1024; i++) {
        int key = random.nextInt(maxSize);
        // ignore deleted & expired
        if (key % 4 == 0 || key % 4 == 1)
            continue;
        GetResult result = targetTable.get(String.valueOf(key).getBytes());
        assertTrue(result.isFound());
    }
    targetTable.close();
    targetTable.delete();
}