Examples with IMapEntry com.ctriposs.sdb.table.IMapEntry used on opensource projects

Example 1 with IMapEntry

use of com.ctriposs.sdb.table.IMapEntry in project sessdb by ppdai.

the class Level0MergerTest method testCase02.

@Test
public void testCase02() throws IOException, ClassNotFoundException {
    HashMapTable[] sourceTables = new HashMapTable[4];
    LevelQueue lq0 = new LevelQueue();
    for (int i = 0; i < 4; i++) {
        sourceTables[i] = new HashMapTable(testDir, SDB.LEVEL0, System.nanoTime() + i);
        sourceTables[i].setCompressionEnabled(true);
        lq0.addFirst(sourceTables[i]);
        assertTrue(sourceTables[i].isImmutable());
    }
    // delete
    int max = AbstractMapTable.INIT_INDEX_ITEMS_PER_TABLE;
    HashMapTable table4 = sourceTables[3];
    int start = 0;
    while (start < max) {
        table4.delete(String.valueOf(start).getBytes());
        start = start + 4;
    }
    // expiration
    HashMapTable table3 = sourceTables[2];
    start = 1;
    while (start < max) {
        table3.put(String.valueOf(start).getBytes(), String.valueOf(start).getBytes(), 200, System.currentTimeMillis());
        start = start + 4;
    }
    // expire table3
    try {
        Thread.sleep(2000);
    } catch (InterruptedException e) {
    // ignore
    }
    // time to live 60 sec
    HashMapTable table2 = sourceTables[1];
    start = 2;
    while (start < max) {
        table2.put(String.valueOf(start).getBytes(), String.valueOf(start).getBytes(), 60 * 1000, System.currentTimeMillis());
        start = start + 4;
    }
    // time to live 120 sec with items that have been updated(table 2), deleted(table 4) and expired(table 3)
    HashMapTable table1 = sourceTables[0];
    start = 0;
    while (start < max) {
        table1.put(String.valueOf(start).getBytes(), String.valueOf(start).getBytes(), 120 * 1000, System.currentTimeMillis());
        start = start + 1;
    }
    LevelQueue lq1 = new LevelQueue();
    Level0Merger.mergeSort(lq0, lq1, 4, testDir, (short) 0);
    for (int i = 0; i < 4; i++) {
        assertTrue(sourceTables[i].isImmutable());
    }
    assertTrue(lq1.size() == 1);
    MMFMapTable targetTable = (MMFMapTable) lq1.poll();
    assertTrue(targetTable.getLevel() == SDB.LEVEL1);
    assertTrue(targetTable.getAppendedSize() == max);
    // validate delete
    start = 0;
    while (start < max) {
        GetResult result = targetTable.get(String.valueOf(start).getBytes());
        assertTrue(result.isFound());
        assertTrue(result.isDeleted());
        start += 4;
    }
    // validate expiration
    start = 1;
    while (start < max) {
        GetResult result = targetTable.get(String.valueOf(start).getBytes());
        assertTrue(result.isFound());
        assertTrue(result.isExpired());
        start += 4;
    }
    // validate ttl 60s
    start = 2;
    while (start < max) {
        GetResult result = targetTable.get(String.valueOf(start).getBytes());
        assertTrue(result.isFound());
        assertTrue(result.getTimeToLive() == 60 * 1000);
        start += 4;
    }
    // validate ttl 120s
    start = 3;
    while (start < max) {
        GetResult result = targetTable.get(String.valueOf(start).getBytes());
        assertTrue(result.isFound());
        assertTrue(result.getTimeToLive() == 120 * 1000);
        start += 4;
    }
    List<String> keyList = new ArrayList<String>();
    for (long i = 0; i < max; i++) {
        keyList.add(String.valueOf(i));
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (int i = 0; i < max; i++) {
        IMapEntry mapEntry = targetTable.getMapEntry(i);
        assertTrue(mapEntry.getIndex() == i);
        assertTrue(new String(mapEntry.getKey()).equals(keyList.get(i)));
        int intKey = Integer.parseInt(new String(mapEntry.getKey()));
        // validate disk space optimization
        if (// deleted
        intKey % 4 == 0)
            assertTrue(Arrays.equals(new byte[] { 0 }, mapEntry.getValue()));
        else if (// expired
        intKey % 4 == 1)
            assertTrue(Arrays.equals(new byte[] { 0 }, mapEntry.getValue()));
        else {
            assertTrue(new String(Snappy.uncompress(mapEntry.getValue())).equals(keyList.get(i)));
        //assertTrue(new String(mapEntry.getValue()).equals(keyList.get(i)));
        }
    }
    Random random = new Random();
    for (int i = 0; i < 1024; i++) {
        long key = random.nextInt(max);
        GetResult result = targetTable.get(String.valueOf(key).getBytes());
        assertTrue(result.isFound());
    }
    targetTable.close();
    targetTable.delete();
}

Example 2 with IMapEntry

use of com.ctriposs.sdb.table.IMapEntry in project sessdb by ppdai.

the class HashMapTableTest method testAppendAndGet.

@Test
public void testAppendAndGet() throws IOException {
    long createdTime = System.nanoTime();
    mapTable = new HashMapTable(testDir, 0, createdTime);
    assertTrue(mapTable.getLevel() == 0);
    assertTrue(mapTable.getCreatedTime() == createdTime);
    assertTrue(mapTable.getAppendedSize() == 0);
    assertTrue(mapTable.isEmpty());
    assertTrue(mapTable.getBackFileSize() == HashMapTable.INIT_INDEX_FILE_SIZE + HashMapTable.INIT_DATA_FILE_SIZE + HashMapTable.INDEX_ITEM_LENGTH);
    mapTable.appendNew("key".getBytes(), "value".getBytes(), 500, System.currentTimeMillis());
    assertTrue(mapTable.getLevel() == 0);
    assertTrue(mapTable.getCreatedTime() == createdTime);
    assertTrue(mapTable.getAppendedSize() == 1);
    assertFalse(mapTable.isEmpty());
    IMapEntry mapEntry = mapTable.getMapEntry(0);
    assertTrue(Arrays.equals("key".getBytes(), mapEntry.getKey()));
    assertTrue(Arrays.equals("value".getBytes(), mapEntry.getValue()));
    assertTrue(500 == mapEntry.getTimeToLive());
    assertTrue(System.currentTimeMillis() >= mapEntry.getCreatedTime());
    assertFalse(mapEntry.isDeleted());
    assertTrue(mapEntry.isInUse());
    try {
        Thread.sleep(1000);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    assertTrue(mapEntry.isExpired());
    mapEntry.markDeleted();
    assertTrue(mapEntry.isDeleted());
}

Example 3 with IMapEntry

use of com.ctriposs.sdb.table.IMapEntry in project sessdb by ppdai.

the class Level0Merger method mergeSort.

public static void mergeSort(LevelQueue source, LevelQueue target, int ways, String dir, short shard) throws IOException, ClassNotFoundException {
    List<HashMapTable> tables = new ArrayList<HashMapTable>(ways);
    source.getReadLock().lock();
    try {
        Iterator<AbstractMapTable> iter = source.descendingIterator();
        for (int i = 0; i < ways; i++) {
            tables.add((HashMapTable) iter.next());
        }
    } finally {
        source.getReadLock().unlock();
    }
    int expectedInsertions = 0;
    for (HashMapTable table : tables) {
        expectedInsertions += table.getRealSize();
    }
    // target table
    MMFMapTable sortedMapTable = new MMFMapTable(dir, shard, SDB.LEVEL1, System.nanoTime(), expectedInsertions, ways);
    PriorityQueue<QueueElement> pq = new PriorityQueue<QueueElement>();
    // build initial heap
    for (HashMapTable table : tables) {
        QueueElement qe = new QueueElement();
        final HashMapTable hmTable = table;
        qe.hashMapTable = hmTable;
        List<Map.Entry<ByteArrayWrapper, InMemIndex>> list = new ArrayList<Map.Entry<ByteArrayWrapper, InMemIndex>>(qe.hashMapTable.getEntrySet());
        Collections.sort(list, new Comparator<Map.Entry<ByteArrayWrapper, InMemIndex>>() {

            @Override
            public int compare(Entry<ByteArrayWrapper, InMemIndex> o1, Entry<ByteArrayWrapper, InMemIndex> o2) {
                IMapEntry mapEntry1 = hmTable.getMapEntry(o1.getValue().getIndex());
                IMapEntry mapEntry2 = hmTable.getMapEntry(o2.getValue().getIndex());
                try {
                    int hash1 = mapEntry1.getKeyHash();
                    int hash2 = mapEntry2.getKeyHash();
                    if (hash1 < hash2)
                        return -1;
                    else if (hash1 > hash2)
                        return 1;
                    else {
                        return o1.getKey().compareTo(o2.getKey());
                    }
                } catch (IOException e) {
                    throw new RuntimeException("Fail to get hash code in map entry", e);
                }
            }
        });
        qe.iterator = list.iterator();
        if (qe.iterator.hasNext()) {
            Map.Entry<ByteArrayWrapper, InMemIndex> me = qe.iterator.next();
            qe.key = me.getKey().getData();
            qe.inMemIndex = me.getValue();
            IMapEntry mapEntry = table.getMapEntry(qe.inMemIndex.getIndex());
            qe.keyHash = mapEntry.getKeyHash();
            pq.add(qe);
        }
    }
    // merge sort
    while (pq.size() > 0) {
        QueueElement qe1 = pq.poll();
        // remove old/stale entries
        while (pq.peek() != null && qe1.keyHash == pq.peek().keyHash && BytesUtil.compare(qe1.key, pq.peek().key) == 0) {
            QueueElement qe2 = pq.poll();
            if (qe2.iterator.hasNext()) {
                Map.Entry<ByteArrayWrapper, InMemIndex> me = qe2.iterator.next();
                qe2.key = me.getKey().getData();
                qe2.inMemIndex = me.getValue();
                IMapEntry mapEntry = qe2.hashMapTable.getMapEntry(qe2.inMemIndex.getIndex());
                qe2.keyHash = mapEntry.getKeyHash();
                pq.add(qe2);
            }
        }
        IMapEntry mapEntry = qe1.hashMapTable.getMapEntry(qe1.inMemIndex.getIndex());
        byte[] value = mapEntry.getValue();
        // disk space optimization
        if (mapEntry.isDeleted() || mapEntry.isExpired()) {
            value = new byte[] { 0 };
        }
        sortedMapTable.appendNew(mapEntry.getKey(), mapEntry.getKeyHash(), value, mapEntry.getTimeToLive(), mapEntry.getCreatedTime(), mapEntry.isDeleted(), mapEntry.isCompressed());
        if (qe1.iterator.hasNext()) {
            Map.Entry<ByteArrayWrapper, InMemIndex> me = qe1.iterator.next();
            qe1.key = me.getKey().getData();
            qe1.inMemIndex = me.getValue();
            IMapEntry mEntry = qe1.hashMapTable.getMapEntry(qe1.inMemIndex.getIndex());
            qe1.keyHash = mEntry.getKeyHash();
            pq.add(qe1);
        }
    }
    // persist metadata
    sortedMapTable.reMap();
    sortedMapTable.saveMetadata();
    // dump to level 1
    source.getWriteLock().lock();
    target.getWriteLock().lock();
    try {
        for (int i = 0; i < ways; i++) {
            source.removeLast();
        }
        for (HashMapTable table : tables) {
            table.markUsable(false);
        }
        sortedMapTable.markUsable(true);
        target.addFirst(sortedMapTable);
    } finally {
        target.getWriteLock().unlock();
        source.getWriteLock().unlock();
    }
    for (HashMapTable table : tables) {
        table.close();
        table.delete();
    }
}

Example 4 with IMapEntry

use of com.ctriposs.sdb.table.IMapEntry in project sessdb by ppdai.

the class Level1Merger method mergeSort.

public static void mergeSort(LevelQueue lq1, LevelQueue lq2, int ways, String dir, short shard) throws IOException, ClassNotFoundException {
    boolean hasLevel2MapTable = lq2.size() > 0;
    List<AbstractMapTable> tables = new ArrayList<AbstractMapTable>(ways);
    lq1.getReadLock().lock();
    try {
        Iterator<AbstractMapTable> iter = lq1.descendingIterator();
        for (int i = 0; i < ways - 1; i++) {
            tables.add(iter.next());
        }
        if (hasLevel2MapTable) {
            tables.add(lq2.get(0));
        } else {
            tables.add(iter.next());
        }
    } finally {
        lq1.getReadLock().unlock();
    }
    long expectedInsertions = 0;
    for (AbstractMapTable table : tables) {
        expectedInsertions += table.getAppendedSize();
    }
    if (expectedInsertions > Integer.MAX_VALUE)
        expectedInsertions = Integer.MAX_VALUE;
    // target table
    AbstractSortedMapTable sortedMapTable = new FCMapTable(dir, shard, SDB.LEVEL2, System.nanoTime(), (int) expectedInsertions);
    PriorityQueue<QueueElement> pq = new PriorityQueue<QueueElement>();
    // build initial heap
    for (AbstractMapTable table : tables) {
        QueueElement qe = new QueueElement();
        qe.sortedMapTable = table;
        qe.size = qe.sortedMapTable.getAppendedSize();
        qe.index = 0;
        qe.queue = new LinkedList<IMapEntry>();
        IMapEntry me = qe.getNextMapEntry();
        if (me != null) {
            qe.key = me.getKey();
            qe.mapEntry = me;
            qe.keyHash = me.getKeyHash();
            pq.add(qe);
        }
    }
    LinkedList<IMapEntry> targetCacheQueue = new LinkedList<IMapEntry>();
    // merge sort
    while (pq.size() > 0) {
        QueueElement qe1 = pq.poll();
        // remove old/stale entries
        while (pq.peek() != null && qe1.keyHash == pq.peek().keyHash && BytesUtil.compare(qe1.key, pq.peek().key) == 0) {
            QueueElement qe2 = pq.poll();
            IMapEntry me = qe2.getNextMapEntry();
            if (me != null) {
                qe2.key = me.getKey();
                qe2.mapEntry = me;
                qe2.keyHash = me.getKeyHash();
                pq.add(qe2);
            }
        }
        // remove deleted or expired entries in final merge sorting
        if (!qe1.mapEntry.isDeleted() && !qe1.mapEntry.isExpired()) {
            targetCacheQueue.add(qe1.mapEntry);
        }
        if (targetCacheQueue.size() >= CACHED_MAP_ENTRIES * DEFAULT_MERGE_WAYS) {
            while (targetCacheQueue.size() > 0) {
                IMapEntry mapEntry = targetCacheQueue.poll();
                byte[] value = mapEntry.getValue();
                // disk space optimization
                if (mapEntry.isExpired()) {
                    continue;
                }
                sortedMapTable.appendNew(mapEntry.getKey(), mapEntry.getKeyHash(), value, mapEntry.getTimeToLive(), mapEntry.getCreatedTime(), mapEntry.isDeleted(), mapEntry.isCompressed());
            }
        }
        IMapEntry me = qe1.getNextMapEntry();
        if (me != null) {
            qe1.key = me.getKey();
            qe1.mapEntry = me;
            qe1.keyHash = me.getKeyHash();
            pq.add(qe1);
        }
    }
    // remaining cached entries
    while (targetCacheQueue.size() > 0) {
        IMapEntry mapEntry = targetCacheQueue.poll();
        byte[] value = mapEntry.getValue();
        // disk space optimization
        if (mapEntry.isExpired()) {
            continue;
        }
        sortedMapTable.appendNew(mapEntry.getKey(), mapEntry.getKeyHash(), value, mapEntry.getTimeToLive(), mapEntry.getCreatedTime(), mapEntry.isDeleted(), mapEntry.isCompressed());
    }
    // persist metadata
    sortedMapTable.reMap();
    sortedMapTable.saveMetadata();
    // switching
    lq1.getWriteLock().lock();
    lq2.getWriteLock().lock();
    try {
        for (int i = 0; i < ways - 1; i++) {
            lq1.removeLast();
        }
        if (hasLevel2MapTable) {
            lq2.removeLast();
        } else {
            lq1.removeLast();
        }
        for (AbstractMapTable table : tables) {
            table.markUsable(false);
        }
        sortedMapTable.markUsable(true);
        lq2.addFirst(sortedMapTable);
    } finally {
        lq2.getWriteLock().unlock();
        lq1.getWriteLock().unlock();
    }
    for (AbstractMapTable table : tables) {
        table.close();
        table.delete();
    }
}

Example 5 with IMapEntry

use of com.ctriposs.sdb.table.IMapEntry in project sessdb by ppdai.

the class Level0MergerTest method testCase01.

@Test
public void testCase01() throws IOException, ClassNotFoundException {
    String value = TestUtil.randomString(128);
    HashMapTable[] sourceTables = new HashMapTable[4];
    LevelQueue lq0 = new LevelQueue();
    for (int i = 0; i < 4; i++) {
        sourceTables[i] = new HashMapTable(testDir, SDB.LEVEL0, System.nanoTime() + i);
        sourceTables[i].setCompressionEnabled(true);
        lq0.addFirst(sourceTables[i]);
        assertTrue(sourceTables[i].isImmutable());
    }
    int totalCount = 0;
    int max = 0;
    for (int i = 0; i < 4; i++) {
        int start = i;
        HashMapTable table = sourceTables[i];
        while (table.put(String.valueOf(start).getBytes(), value.getBytes(), AbstractMapTable.NO_TIMEOUT, System.currentTimeMillis(), false)) {
            totalCount++;
            if (start > max)
                max = start;
            start = start + 4;
        }
    }
    LevelQueue lq1 = new LevelQueue();
    long start = System.currentTimeMillis();
    Level0Merger.mergeSort(lq0, lq1, 4, testDir, (short) 1);
    long end = System.currentTimeMillis();
    System.out.println("Time spent to merge " + totalCount + " items in 4 ways  is " + (end - start) / 1000 + "s");
    for (int i = 0; i < 4; i++) {
        assertTrue(sourceTables[i].isImmutable());
    }
    assertTrue(lq1.size() == 1);
    MMFMapTable targetTable = (MMFMapTable) lq1.poll();
    assertTrue(targetTable.getLevel() == SDB.LEVEL1);
    assertTrue(targetTable.getAppendedSize() == totalCount);
    List<String> keyList = new ArrayList<String>();
    for (long i = 0; i < totalCount; i++) {
        keyList.add(String.valueOf(i));
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (int i = 0; i < totalCount; i++) {
        IMapEntry mapEntry = targetTable.getMapEntry(i);
        assertTrue(mapEntry.getIndex() == i);
        assertTrue(new String(mapEntry.getKey()).equals(keyList.get(i)));
        assertTrue(new String(Snappy.uncompress(mapEntry.getValue())).equals(value));
    }
    start = System.currentTimeMillis();
    Random random = new Random();
    for (int i = 0; i < 1024; i++) {
        long key = random.nextInt(totalCount);
        GetResult result = targetTable.get(String.valueOf(key).getBytes());
        assertTrue(result.isFound());
    }
    end = System.currentTimeMillis();
    System.out.println("Time to lookup 1024 random key in the target table is " + (end - start) + "ms");
    targetTable.close();
    targetTable.delete();
}