Examples with AbstractMapTable com.ctriposs.sdb.table.AbstractMapTable used on opensource projects

Example 1 with AbstractMapTable

use of com.ctriposs.sdb.table.AbstractMapTable in project sessdb by ppdai.

the class SDB method close.

@Override
public void close() throws IOException {
    if (closed)
        return;
    fileStatsCollector.setStop();
    for (int i = 0; i < config.getShardNumber(); i++) {
        this.activeInMemTables[i].close();
    }
    for (int i = 0; i < config.getShardNumber(); i++) {
        this.level0Mergers[i].setStop();
        this.level1Mergers[i].setStop();
    }
    for (int i = 0; i < config.getShardNumber(); i++) {
        try {
            log.info("Shard " + i + " waiting level 0 & 1 merge threads to exit...");
            this.countDownLatches[i].await();
        } catch (InterruptedException e) {
        // ignore;
        }
    }
    for (int i = 0; i < config.getShardNumber(); i++) {
        for (int j = 0; j <= MAX_LEVEL; j++) {
            LevelQueue lq = this.levelQueueLists[i].get(j);
            for (AbstractMapTable table : lq) {
                table.close();
            }
        }
    }
    closed = true;
    log.info("DB Closed.");
}

Example 2 with AbstractMapTable

use of com.ctriposs.sdb.table.AbstractMapTable in project sessdb by ppdai.

the class Level0Merger method mergeSort.

public static void mergeSort(LevelQueue source, LevelQueue target, int ways, String dir, short shard) throws IOException, ClassNotFoundException {
    List<HashMapTable> tables = new ArrayList<HashMapTable>(ways);
    source.getReadLock().lock();
    try {
        Iterator<AbstractMapTable> iter = source.descendingIterator();
        for (int i = 0; i < ways; i++) {
            tables.add((HashMapTable) iter.next());
        }
    } finally {
        source.getReadLock().unlock();
    }
    int expectedInsertions = 0;
    for (HashMapTable table : tables) {
        expectedInsertions += table.getRealSize();
    }
    // target table
    MMFMapTable sortedMapTable = new MMFMapTable(dir, shard, SDB.LEVEL1, System.nanoTime(), expectedInsertions, ways);
    PriorityQueue<QueueElement> pq = new PriorityQueue<QueueElement>();
    // build initial heap
    for (HashMapTable table : tables) {
        QueueElement qe = new QueueElement();
        final HashMapTable hmTable = table;
        qe.hashMapTable = hmTable;
        List<Map.Entry<ByteArrayWrapper, InMemIndex>> list = new ArrayList<Map.Entry<ByteArrayWrapper, InMemIndex>>(qe.hashMapTable.getEntrySet());
        Collections.sort(list, new Comparator<Map.Entry<ByteArrayWrapper, InMemIndex>>() {

            @Override
            public int compare(Entry<ByteArrayWrapper, InMemIndex> o1, Entry<ByteArrayWrapper, InMemIndex> o2) {
                IMapEntry mapEntry1 = hmTable.getMapEntry(o1.getValue().getIndex());
                IMapEntry mapEntry2 = hmTable.getMapEntry(o2.getValue().getIndex());
                try {
                    int hash1 = mapEntry1.getKeyHash();
                    int hash2 = mapEntry2.getKeyHash();
                    if (hash1 < hash2)
                        return -1;
                    else if (hash1 > hash2)
                        return 1;
                    else {
                        return o1.getKey().compareTo(o2.getKey());
                    }
                } catch (IOException e) {
                    throw new RuntimeException("Fail to get hash code in map entry", e);
                }
            }
        });
        qe.iterator = list.iterator();
        if (qe.iterator.hasNext()) {
            Map.Entry<ByteArrayWrapper, InMemIndex> me = qe.iterator.next();
            qe.key = me.getKey().getData();
            qe.inMemIndex = me.getValue();
            IMapEntry mapEntry = table.getMapEntry(qe.inMemIndex.getIndex());
            qe.keyHash = mapEntry.getKeyHash();
            pq.add(qe);
        }
    }
    // merge sort
    while (pq.size() > 0) {
        QueueElement qe1 = pq.poll();
        // remove old/stale entries
        while (pq.peek() != null && qe1.keyHash == pq.peek().keyHash && BytesUtil.compare(qe1.key, pq.peek().key) == 0) {
            QueueElement qe2 = pq.poll();
            if (qe2.iterator.hasNext()) {
                Map.Entry<ByteArrayWrapper, InMemIndex> me = qe2.iterator.next();
                qe2.key = me.getKey().getData();
                qe2.inMemIndex = me.getValue();
                IMapEntry mapEntry = qe2.hashMapTable.getMapEntry(qe2.inMemIndex.getIndex());
                qe2.keyHash = mapEntry.getKeyHash();
                pq.add(qe2);
            }
        }
        IMapEntry mapEntry = qe1.hashMapTable.getMapEntry(qe1.inMemIndex.getIndex());
        byte[] value = mapEntry.getValue();
        // disk space optimization
        if (mapEntry.isDeleted() || mapEntry.isExpired()) {
            value = new byte[] { 0 };
        }
        sortedMapTable.appendNew(mapEntry.getKey(), mapEntry.getKeyHash(), value, mapEntry.getTimeToLive(), mapEntry.getCreatedTime(), mapEntry.isDeleted(), mapEntry.isCompressed());
        if (qe1.iterator.hasNext()) {
            Map.Entry<ByteArrayWrapper, InMemIndex> me = qe1.iterator.next();
            qe1.key = me.getKey().getData();
            qe1.inMemIndex = me.getValue();
            IMapEntry mEntry = qe1.hashMapTable.getMapEntry(qe1.inMemIndex.getIndex());
            qe1.keyHash = mEntry.getKeyHash();
            pq.add(qe1);
        }
    }
    // persist metadata
    sortedMapTable.reMap();
    sortedMapTable.saveMetadata();
    // dump to level 1
    source.getWriteLock().lock();
    target.getWriteLock().lock();
    try {
        for (int i = 0; i < ways; i++) {
            source.removeLast();
        }
        for (HashMapTable table : tables) {
            table.markUsable(false);
        }
        sortedMapTable.markUsable(true);
        target.addFirst(sortedMapTable);
    } finally {
        target.getWriteLock().unlock();
        source.getWriteLock().unlock();
    }
    for (HashMapTable table : tables) {
        table.close();
        table.delete();
    }
}

Example 3 with AbstractMapTable

use of com.ctriposs.sdb.table.AbstractMapTable in project sessdb by ppdai.

the class FileStatsCollector method run.

@Override
public void run() {
    while (!stop) {
        try {
            for (int level = 0; level <= SDB.MAX_LEVEL; ++level) {
                long fileSize = 0;
                int fileCount = 0;
                for (int shard = 0; shard < levelQueueLists.length; ++shard) {
                    LevelQueue queue = levelQueueLists[shard].get(level);
                    queue.getReadLock().lock();
                    try {
                        for (AbstractMapTable table : queue) {
                            fileSize += table.getBackFileSize();
                        }
                        fileCount += queue.size();
                    } finally {
                        queue.getReadLock().unlock();
                    }
                }
                stats.recordFileStats(level, fileCount, fileSize);
            }
            Thread.sleep(MAX_SLEEP_TIME);
        } catch (Exception ex) {
            log.error("Error occurred in the file stats collector", ex);
        }
    }
}

Example 4 with AbstractMapTable

use of com.ctriposs.sdb.table.AbstractMapTable in project sessdb by ppdai.

the class Level1Merger method mergeSort.

public static void mergeSort(LevelQueue lq1, LevelQueue lq2, int ways, String dir, short shard) throws IOException, ClassNotFoundException {
    boolean hasLevel2MapTable = lq2.size() > 0;
    List<AbstractMapTable> tables = new ArrayList<AbstractMapTable>(ways);
    lq1.getReadLock().lock();
    try {
        Iterator<AbstractMapTable> iter = lq1.descendingIterator();
        for (int i = 0; i < ways - 1; i++) {
            tables.add(iter.next());
        }
        if (hasLevel2MapTable) {
            tables.add(lq2.get(0));
        } else {
            tables.add(iter.next());
        }
    } finally {
        lq1.getReadLock().unlock();
    }
    long expectedInsertions = 0;
    for (AbstractMapTable table : tables) {
        expectedInsertions += table.getAppendedSize();
    }
    if (expectedInsertions > Integer.MAX_VALUE)
        expectedInsertions = Integer.MAX_VALUE;
    // target table
    AbstractSortedMapTable sortedMapTable = new FCMapTable(dir, shard, SDB.LEVEL2, System.nanoTime(), (int) expectedInsertions);
    PriorityQueue<QueueElement> pq = new PriorityQueue<QueueElement>();
    // build initial heap
    for (AbstractMapTable table : tables) {
        QueueElement qe = new QueueElement();
        qe.sortedMapTable = table;
        qe.size = qe.sortedMapTable.getAppendedSize();
        qe.index = 0;
        qe.queue = new LinkedList<IMapEntry>();
        IMapEntry me = qe.getNextMapEntry();
        if (me != null) {
            qe.key = me.getKey();
            qe.mapEntry = me;
            qe.keyHash = me.getKeyHash();
            pq.add(qe);
        }
    }
    LinkedList<IMapEntry> targetCacheQueue = new LinkedList<IMapEntry>();
    // merge sort
    while (pq.size() > 0) {
        QueueElement qe1 = pq.poll();
        // remove old/stale entries
        while (pq.peek() != null && qe1.keyHash == pq.peek().keyHash && BytesUtil.compare(qe1.key, pq.peek().key) == 0) {
            QueueElement qe2 = pq.poll();
            IMapEntry me = qe2.getNextMapEntry();
            if (me != null) {
                qe2.key = me.getKey();
                qe2.mapEntry = me;
                qe2.keyHash = me.getKeyHash();
                pq.add(qe2);
            }
        }
        // remove deleted or expired entries in final merge sorting
        if (!qe1.mapEntry.isDeleted() && !qe1.mapEntry.isExpired()) {
            targetCacheQueue.add(qe1.mapEntry);
        }
        if (targetCacheQueue.size() >= CACHED_MAP_ENTRIES * DEFAULT_MERGE_WAYS) {
            while (targetCacheQueue.size() > 0) {
                IMapEntry mapEntry = targetCacheQueue.poll();
                byte[] value = mapEntry.getValue();
                // disk space optimization
                if (mapEntry.isExpired()) {
                    continue;
                }
                sortedMapTable.appendNew(mapEntry.getKey(), mapEntry.getKeyHash(), value, mapEntry.getTimeToLive(), mapEntry.getCreatedTime(), mapEntry.isDeleted(), mapEntry.isCompressed());
            }
        }
        IMapEntry me = qe1.getNextMapEntry();
        if (me != null) {
            qe1.key = me.getKey();
            qe1.mapEntry = me;
            qe1.keyHash = me.getKeyHash();
            pq.add(qe1);
        }
    }
    // remaining cached entries
    while (targetCacheQueue.size() > 0) {
        IMapEntry mapEntry = targetCacheQueue.poll();
        byte[] value = mapEntry.getValue();
        // disk space optimization
        if (mapEntry.isExpired()) {
            continue;
        }
        sortedMapTable.appendNew(mapEntry.getKey(), mapEntry.getKeyHash(), value, mapEntry.getTimeToLive(), mapEntry.getCreatedTime(), mapEntry.isDeleted(), mapEntry.isCompressed());
    }
    // persist metadata
    sortedMapTable.reMap();
    sortedMapTable.saveMetadata();
    // switching
    lq1.getWriteLock().lock();
    lq2.getWriteLock().lock();
    try {
        for (int i = 0; i < ways - 1; i++) {
            lq1.removeLast();
        }
        if (hasLevel2MapTable) {
            lq2.removeLast();
        } else {
            lq1.removeLast();
        }
        for (AbstractMapTable table : tables) {
            table.markUsable(false);
        }
        sortedMapTable.markUsable(true);
        lq2.addFirst(sortedMapTable);
    } finally {
        lq2.getWriteLock().unlock();
        lq1.getWriteLock().unlock();
    }
    for (AbstractMapTable table : tables) {
        table.close();
        table.delete();
    }
}

Example 5 with AbstractMapTable

use of com.ctriposs.sdb.table.AbstractMapTable in project sessdb by ppdai.

the class SDB method get.

/**
	 * Get value in the DB with specific key
	 *
	 * @param key map entry key
	 * @return non-null value if the entry exists, not deleted or expired.
	 * null value if the entry does not exist, or exists but deleted or expired.
	 */
public byte[] get(byte[] key) {
    Preconditions.checkArgument(key != null && key.length > 0, "key is empty");
    ensureNotClosed();
    long start = System.nanoTime();
    int reachedLevel = INMEM_LEVEL;
    try {
        short shard = this.getShard(key);
        // check active hashmap table first
        GetResult result = this.activeInMemTables[shard].get(key);
        if (result.isFound()) {
            if (!result.isDeleted() && !result.isExpired()) {
                return result.getValue();
            } else {
                // deleted or expired
                return null;
            }
        } else {
            // check level0 hashmap tables
            reachedLevel = LEVEL0;
            LevelQueue lq0 = levelQueueLists[shard].get(LEVEL0);
            lq0.getReadLock().lock();
            try {
                if (lq0 != null && lq0.size() > 0) {
                    for (AbstractMapTable table : lq0) {
                        result = table.get(key);
                        if (result.isFound())
                            break;
                    }
                }
            } finally {
                lq0.getReadLock().unlock();
            }
            if (result.isFound()) {
                if (!result.isDeleted() && !result.isExpired()) {
                    if (result.getLevel() == SDB.LEVEL2 && this.config.isLocalityEnabled()) {
                        // keep locality
                        this.put(key, result.getValue(), result.getTimeToLive(), result.getCreatedTime(), false);
                    }
                    return result.getValue();
                } else {
                    // deleted or expired
                    return null;
                }
            }
            // check level 1-2 on disk sorted tables
            searchLevel12: {
                for (int level = 1; level <= MAX_LEVEL; level++) {
                    reachedLevel = level;
                    LevelQueue lq = levelQueueLists[shard].get(level);
                    lq.getReadLock().lock();
                    try {
                        if (lq.size() > 0) {
                            for (AbstractMapTable table : lq) {
                                result = table.get(key);
                                if (result.isFound())
                                    break searchLevel12;
                            }
                        }
                    } finally {
                        lq.getReadLock().unlock();
                    }
                }
            }
            if (result.isFound()) {
                if (!result.isDeleted() && !result.isExpired()) {
                    if (result.getLevel() == SDB.LEVEL2 && this.config.isLocalityEnabled()) {
                        // keep locality
                        this.put(key, result.getValue(), result.getTimeToLive(), result.getCreatedTime(), false);
                    }
                    return result.getValue();
                } else {
                    // deleted or expired
                    return null;
                }
            }
        }
    } catch (IOException ioe) {
        stats.recordDBError(Operations.GET);
        throw new RuntimeException("Fail to get value by key, IOException occurr", ioe);
    } finally {
        stats.recordDBOperation(Operations.GET, reachedLevel, System.nanoTime() - start);
    }
    // no luck
    return null;
}