Examples with HashMapTable com.ctriposs.sdb.table.HashMapTable used on opensource projects

Example 6 with HashMapTable

use of com.ctriposs.sdb.table.HashMapTable in project sessdb by ppdai.

the class HashMapTableTest method testAppendAndGet.

@Test
public void testAppendAndGet() throws IOException {
    long createdTime = System.nanoTime();
    mapTable = new HashMapTable(testDir, 0, createdTime);
    assertTrue(mapTable.getLevel() == 0);
    assertTrue(mapTable.getCreatedTime() == createdTime);
    assertTrue(mapTable.getAppendedSize() == 0);
    assertTrue(mapTable.isEmpty());
    assertTrue(mapTable.getBackFileSize() == HashMapTable.INIT_INDEX_FILE_SIZE + HashMapTable.INIT_DATA_FILE_SIZE + HashMapTable.INDEX_ITEM_LENGTH);
    mapTable.appendNew("key".getBytes(), "value".getBytes(), 500, System.currentTimeMillis());
    assertTrue(mapTable.getLevel() == 0);
    assertTrue(mapTable.getCreatedTime() == createdTime);
    assertTrue(mapTable.getAppendedSize() == 1);
    assertFalse(mapTable.isEmpty());
    IMapEntry mapEntry = mapTable.getMapEntry(0);
    assertTrue(Arrays.equals("key".getBytes(), mapEntry.getKey()));
    assertTrue(Arrays.equals("value".getBytes(), mapEntry.getValue()));
    assertTrue(500 == mapEntry.getTimeToLive());
    assertTrue(System.currentTimeMillis() >= mapEntry.getCreatedTime());
    assertFalse(mapEntry.isDeleted());
    assertTrue(mapEntry.isInUse());
    try {
        Thread.sleep(1000);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    assertTrue(mapEntry.isExpired());
    mapEntry.markDeleted();
    assertTrue(mapEntry.isDeleted());
}

Example 7 with HashMapTable

use of com.ctriposs.sdb.table.HashMapTable in project sessdb by ppdai.

the class Level0Merger method mergeSort.

public static void mergeSort(LevelQueue source, LevelQueue target, int ways, String dir, short shard) throws IOException, ClassNotFoundException {
    List<HashMapTable> tables = new ArrayList<HashMapTable>(ways);
    source.getReadLock().lock();
    try {
        Iterator<AbstractMapTable> iter = source.descendingIterator();
        for (int i = 0; i < ways; i++) {
            tables.add((HashMapTable) iter.next());
        }
    } finally {
        source.getReadLock().unlock();
    }
    int expectedInsertions = 0;
    for (HashMapTable table : tables) {
        expectedInsertions += table.getRealSize();
    }
    // target table
    MMFMapTable sortedMapTable = new MMFMapTable(dir, shard, SDB.LEVEL1, System.nanoTime(), expectedInsertions, ways);
    PriorityQueue<QueueElement> pq = new PriorityQueue<QueueElement>();
    // build initial heap
    for (HashMapTable table : tables) {
        QueueElement qe = new QueueElement();
        final HashMapTable hmTable = table;
        qe.hashMapTable = hmTable;
        List<Map.Entry<ByteArrayWrapper, InMemIndex>> list = new ArrayList<Map.Entry<ByteArrayWrapper, InMemIndex>>(qe.hashMapTable.getEntrySet());
        Collections.sort(list, new Comparator<Map.Entry<ByteArrayWrapper, InMemIndex>>() {

            @Override
            public int compare(Entry<ByteArrayWrapper, InMemIndex> o1, Entry<ByteArrayWrapper, InMemIndex> o2) {
                IMapEntry mapEntry1 = hmTable.getMapEntry(o1.getValue().getIndex());
                IMapEntry mapEntry2 = hmTable.getMapEntry(o2.getValue().getIndex());
                try {
                    int hash1 = mapEntry1.getKeyHash();
                    int hash2 = mapEntry2.getKeyHash();
                    if (hash1 < hash2)
                        return -1;
                    else if (hash1 > hash2)
                        return 1;
                    else {
                        return o1.getKey().compareTo(o2.getKey());
                    }
                } catch (IOException e) {
                    throw new RuntimeException("Fail to get hash code in map entry", e);
                }
            }
        });
        qe.iterator = list.iterator();
        if (qe.iterator.hasNext()) {
            Map.Entry<ByteArrayWrapper, InMemIndex> me = qe.iterator.next();
            qe.key = me.getKey().getData();
            qe.inMemIndex = me.getValue();
            IMapEntry mapEntry = table.getMapEntry(qe.inMemIndex.getIndex());
            qe.keyHash = mapEntry.getKeyHash();
            pq.add(qe);
        }
    }
    // merge sort
    while (pq.size() > 0) {
        QueueElement qe1 = pq.poll();
        // remove old/stale entries
        while (pq.peek() != null && qe1.keyHash == pq.peek().keyHash && BytesUtil.compare(qe1.key, pq.peek().key) == 0) {
            QueueElement qe2 = pq.poll();
            if (qe2.iterator.hasNext()) {
                Map.Entry<ByteArrayWrapper, InMemIndex> me = qe2.iterator.next();
                qe2.key = me.getKey().getData();
                qe2.inMemIndex = me.getValue();
                IMapEntry mapEntry = qe2.hashMapTable.getMapEntry(qe2.inMemIndex.getIndex());
                qe2.keyHash = mapEntry.getKeyHash();
                pq.add(qe2);
            }
        }
        IMapEntry mapEntry = qe1.hashMapTable.getMapEntry(qe1.inMemIndex.getIndex());
        byte[] value = mapEntry.getValue();
        // disk space optimization
        if (mapEntry.isDeleted() || mapEntry.isExpired()) {
            value = new byte[] { 0 };
        }
        sortedMapTable.appendNew(mapEntry.getKey(), mapEntry.getKeyHash(), value, mapEntry.getTimeToLive(), mapEntry.getCreatedTime(), mapEntry.isDeleted(), mapEntry.isCompressed());
        if (qe1.iterator.hasNext()) {
            Map.Entry<ByteArrayWrapper, InMemIndex> me = qe1.iterator.next();
            qe1.key = me.getKey().getData();
            qe1.inMemIndex = me.getValue();
            IMapEntry mEntry = qe1.hashMapTable.getMapEntry(qe1.inMemIndex.getIndex());
            qe1.keyHash = mEntry.getKeyHash();
            pq.add(qe1);
        }
    }
    // persist metadata
    sortedMapTable.reMap();
    sortedMapTable.saveMetadata();
    // dump to level 1
    source.getWriteLock().lock();
    target.getWriteLock().lock();
    try {
        for (int i = 0; i < ways; i++) {
            source.removeLast();
        }
        for (HashMapTable table : tables) {
            table.markUsable(false);
        }
        sortedMapTable.markUsable(true);
        target.addFirst(sortedMapTable);
    } finally {
        target.getWriteLock().unlock();
        source.getWriteLock().unlock();
    }
    for (HashMapTable table : tables) {
        table.close();
        table.delete();
    }
}

Example 8 with HashMapTable

use of com.ctriposs.sdb.table.HashMapTable in project sessdb by ppdai.

the class SDB method loadMapTables.

private void loadMapTables() throws IOException, ClassNotFoundException {
    File dirFile = new File(dir);
    if (!dirFile.exists()) {
        dirFile.mkdirs();
    }
    String[] fileNames = dirFile.list(new FilenameFilter() {

        @Override
        public boolean accept(File dir, String filename) {
            if (filename.endsWith(AbstractMapTable.INDEX_FILE_SUFFIX))
                return true;
            return false;
        }
    });
    // new DB, setup new active map table
    if (fileNames == null || fileNames.length == 0) {
        for (short i = 0; i < this.config.getShardNumber(); i++) {
            this.activeInMemTables[i] = new HashMapTable(dir, i, LEVEL0, System.nanoTime());
            this.activeInMemTables[i].markUsable(true);
            // mutable
            this.activeInMemTables[i].markImmutable(false);
            this.activeInMemTables[i].setCompressionEnabled(this.config.isCompressionEnabled());
        }
        return;
    }
    PriorityQueue<AbstractMapTable> pq = new PriorityQueue<AbstractMapTable>();
    for (String fileName : fileNames) {
        int dotIndex = fileName.lastIndexOf(".");
        if (dotIndex > 0) {
            fileName = fileName.substring(0, dotIndex);
        }
        String[] parts = fileName.split("-");
        Preconditions.checkArgument(parts != null && parts.length == 3, "on-disk table file names corrupted!");
        int level = Integer.parseInt(parts[1]);
        if (level == LEVEL0) {
            pq.add(new HashMapTable(dir, fileName));
        } else if (level == LEVEL1) {
            pq.add(new MMFMapTable(dir, fileName));
        } else {
            pq.add(new FCMapTable(dir, fileName));
        }
    }
    Preconditions.checkArgument(pq.size() > 0, "on-disk table file names corrupted!");
    // setup active map table
    for (int i = 0; i < this.config.getShardNumber(); i++) {
        AbstractMapTable table = pq.poll();
        Preconditions.checkArgument(table.getLevel() == 0, "on-disk table file names corrupted, no level 0 map tables");
        this.activeInMemTables[table.getShard()] = (HashMapTable) table;
        this.activeInMemTables[table.getShard()].markUsable(true);
        // mutable
        this.activeInMemTables[table.getShard()].markImmutable(false);
        this.activeInMemTables[table.getShard()].setCompressionEnabled(this.config.isCompressionEnabled());
    }
    while (!pq.isEmpty()) {
        AbstractMapTable table = pq.poll();
        if (table.isUsable()) {
            int level = table.getLevel();
            LevelQueue lq = levelQueueLists[table.getShard()].get(level);
            lq.addLast(table);
        } else {
            // garbage
            table.close();
            table.delete();
        }
    }
}

Example 9 with HashMapTable

use of com.ctriposs.sdb.table.HashMapTable in project sessdb by ppdai.

the class SDB method put.

private void put(byte[] key, byte[] value, long timeToLive, long createdTime, boolean isDelete) {
    Preconditions.checkArgument(key != null && key.length > 0, "key is empty");
    Preconditions.checkArgument(value != null && value.length > 0, "value is empty");
    ensureNotClosed();
    long start = System.nanoTime();
    String operation = isDelete ? Operations.DELETE : Operations.PUT;
    try {
        short shard = this.getShard(key);
        boolean success = this.activeInMemTables[shard].put(key, value, timeToLive, createdTime, isDelete);
        if (!success) {
            // overflow
            synchronized (activeInMemTableCreationLocks[shard]) {
                // other thread may have done the creation work
                success = this.activeInMemTables[shard].put(key, value, timeToLive, createdTime, isDelete);
                if (!success) {
                    // move to level queue 0
                    this.activeInMemTables[shard].markImmutable(true);
                    LevelQueue lq0 = this.levelQueueLists[shard].get(LEVEL0);
                    lq0.getWriteLock().lock();
                    try {
                        lq0.addFirst(this.activeInMemTables[shard]);
                    } finally {
                        lq0.getWriteLock().unlock();
                    }
                    @SuppressWarnings("resource") HashMapTable tempTable = new HashMapTable(dir, shard, LEVEL0, System.nanoTime());
                    tempTable.markUsable(true);
                    //mutable
                    tempTable.markImmutable(false);
                    tempTable.put(key, value, timeToLive, createdTime, isDelete);
                    // switch on
                    this.activeInMemTables[shard] = tempTable;
                }
            }
        }
    } catch (IOException ioe) {
        stats.recordDBError(operation);
        if (isDelete) {
            throw new RuntimeException("Fail to delete key, IOException occurr", ioe);
        }
        throw new RuntimeException("Fail to put key & value, IOException occurr", ioe);
    } finally {
        stats.recordDBOperation(operation, INMEM_LEVEL, System.nanoTime() - start);
    }
}

Example 10 with HashMapTable

use of com.ctriposs.sdb.table.HashMapTable in project sessdb by ppdai.

the class Level0MergerTest method testCase01.

@Test
public void testCase01() throws IOException, ClassNotFoundException {
    String value = TestUtil.randomString(128);
    HashMapTable[] sourceTables = new HashMapTable[4];
    LevelQueue lq0 = new LevelQueue();
    for (int i = 0; i < 4; i++) {
        sourceTables[i] = new HashMapTable(testDir, SDB.LEVEL0, System.nanoTime() + i);
        sourceTables[i].setCompressionEnabled(true);
        lq0.addFirst(sourceTables[i]);
        assertTrue(sourceTables[i].isImmutable());
    }
    int totalCount = 0;
    int max = 0;
    for (int i = 0; i < 4; i++) {
        int start = i;
        HashMapTable table = sourceTables[i];
        while (table.put(String.valueOf(start).getBytes(), value.getBytes(), AbstractMapTable.NO_TIMEOUT, System.currentTimeMillis(), false)) {
            totalCount++;
            if (start > max)
                max = start;
            start = start + 4;
        }
    }
    LevelQueue lq1 = new LevelQueue();
    long start = System.currentTimeMillis();
    Level0Merger.mergeSort(lq0, lq1, 4, testDir, (short) 1);
    long end = System.currentTimeMillis();
    System.out.println("Time spent to merge " + totalCount + " items in 4 ways  is " + (end - start) / 1000 + "s");
    for (int i = 0; i < 4; i++) {
        assertTrue(sourceTables[i].isImmutable());
    }
    assertTrue(lq1.size() == 1);
    MMFMapTable targetTable = (MMFMapTable) lq1.poll();
    assertTrue(targetTable.getLevel() == SDB.LEVEL1);
    assertTrue(targetTable.getAppendedSize() == totalCount);
    List<String> keyList = new ArrayList<String>();
    for (long i = 0; i < totalCount; i++) {
        keyList.add(String.valueOf(i));
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (int i = 0; i < totalCount; i++) {
        IMapEntry mapEntry = targetTable.getMapEntry(i);
        assertTrue(mapEntry.getIndex() == i);
        assertTrue(new String(mapEntry.getKey()).equals(keyList.get(i)));
        assertTrue(new String(Snappy.uncompress(mapEntry.getValue())).equals(value));
    }
    start = System.currentTimeMillis();
    Random random = new Random();
    for (int i = 0; i < 1024; i++) {
        long key = random.nextInt(totalCount);
        GetResult result = targetTable.get(String.valueOf(key).getBytes());
        assertTrue(result.isFound());
    }
    end = System.currentTimeMillis();
    System.out.println("Time to lookup 1024 random key in the target table is " + (end - start) + "ms");
    targetTable.close();
    targetTable.delete();
}