Examples with MMFMapTable com.ctriposs.sdb.table.MMFMapTable used on opensource projects

Example 6 with MMFMapTable

use of com.ctriposs.sdb.table.MMFMapTable in project sessdb by ppdai.

the class SDB method loadMapTables.

private void loadMapTables() throws IOException, ClassNotFoundException {
    File dirFile = new File(dir);
    if (!dirFile.exists()) {
        dirFile.mkdirs();
    }
    String[] fileNames = dirFile.list(new FilenameFilter() {

        @Override
        public boolean accept(File dir, String filename) {
            if (filename.endsWith(AbstractMapTable.INDEX_FILE_SUFFIX))
                return true;
            return false;
        }
    });
    // new DB, setup new active map table
    if (fileNames == null || fileNames.length == 0) {
        for (short i = 0; i < this.config.getShardNumber(); i++) {
            this.activeInMemTables[i] = new HashMapTable(dir, i, LEVEL0, System.nanoTime());
            this.activeInMemTables[i].markUsable(true);
            // mutable
            this.activeInMemTables[i].markImmutable(false);
            this.activeInMemTables[i].setCompressionEnabled(this.config.isCompressionEnabled());
        }
        return;
    }
    PriorityQueue<AbstractMapTable> pq = new PriorityQueue<AbstractMapTable>();
    for (String fileName : fileNames) {
        int dotIndex = fileName.lastIndexOf(".");
        if (dotIndex > 0) {
            fileName = fileName.substring(0, dotIndex);
        }
        String[] parts = fileName.split("-");
        Preconditions.checkArgument(parts != null && parts.length == 3, "on-disk table file names corrupted!");
        int level = Integer.parseInt(parts[1]);
        if (level == LEVEL0) {
            pq.add(new HashMapTable(dir, fileName));
        } else if (level == LEVEL1) {
            pq.add(new MMFMapTable(dir, fileName));
        } else {
            pq.add(new FCMapTable(dir, fileName));
        }
    }
    Preconditions.checkArgument(pq.size() > 0, "on-disk table file names corrupted!");
    // setup active map table
    for (int i = 0; i < this.config.getShardNumber(); i++) {
        AbstractMapTable table = pq.poll();
        Preconditions.checkArgument(table.getLevel() == 0, "on-disk table file names corrupted, no level 0 map tables");
        this.activeInMemTables[table.getShard()] = (HashMapTable) table;
        this.activeInMemTables[table.getShard()].markUsable(true);
        // mutable
        this.activeInMemTables[table.getShard()].markImmutable(false);
        this.activeInMemTables[table.getShard()].setCompressionEnabled(this.config.isCompressionEnabled());
    }
    while (!pq.isEmpty()) {
        AbstractMapTable table = pq.poll();
        if (table.isUsable()) {
            int level = table.getLevel();
            LevelQueue lq = levelQueueLists[table.getShard()].get(level);
            lq.addLast(table);
        } else {
            // garbage
            table.close();
            table.delete();
        }
    }
}

Example 7 with MMFMapTable

use of com.ctriposs.sdb.table.MMFMapTable in project sessdb by ppdai.

the class Level0MergerTest method testCase01.

@Test
public void testCase01() throws IOException, ClassNotFoundException {
    String value = TestUtil.randomString(128);
    HashMapTable[] sourceTables = new HashMapTable[4];
    LevelQueue lq0 = new LevelQueue();
    for (int i = 0; i < 4; i++) {
        sourceTables[i] = new HashMapTable(testDir, SDB.LEVEL0, System.nanoTime() + i);
        sourceTables[i].setCompressionEnabled(true);
        lq0.addFirst(sourceTables[i]);
        assertTrue(sourceTables[i].isImmutable());
    }
    int totalCount = 0;
    int max = 0;
    for (int i = 0; i < 4; i++) {
        int start = i;
        HashMapTable table = sourceTables[i];
        while (table.put(String.valueOf(start).getBytes(), value.getBytes(), AbstractMapTable.NO_TIMEOUT, System.currentTimeMillis(), false)) {
            totalCount++;
            if (start > max)
                max = start;
            start = start + 4;
        }
    }
    LevelQueue lq1 = new LevelQueue();
    long start = System.currentTimeMillis();
    Level0Merger.mergeSort(lq0, lq1, 4, testDir, (short) 1);
    long end = System.currentTimeMillis();
    System.out.println("Time spent to merge " + totalCount + " items in 4 ways  is " + (end - start) / 1000 + "s");
    for (int i = 0; i < 4; i++) {
        assertTrue(sourceTables[i].isImmutable());
    }
    assertTrue(lq1.size() == 1);
    MMFMapTable targetTable = (MMFMapTable) lq1.poll();
    assertTrue(targetTable.getLevel() == SDB.LEVEL1);
    assertTrue(targetTable.getAppendedSize() == totalCount);
    List<String> keyList = new ArrayList<String>();
    for (long i = 0; i < totalCount; i++) {
        keyList.add(String.valueOf(i));
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (int i = 0; i < totalCount; i++) {
        IMapEntry mapEntry = targetTable.getMapEntry(i);
        assertTrue(mapEntry.getIndex() == i);
        assertTrue(new String(mapEntry.getKey()).equals(keyList.get(i)));
        assertTrue(new String(Snappy.uncompress(mapEntry.getValue())).equals(value));
    }
    start = System.currentTimeMillis();
    Random random = new Random();
    for (int i = 0; i < 1024; i++) {
        long key = random.nextInt(totalCount);
        GetResult result = targetTable.get(String.valueOf(key).getBytes());
        assertTrue(result.isFound());
    }
    end = System.currentTimeMillis();
    System.out.println("Time to lookup 1024 random key in the target table is " + (end - start) + "ms");
    targetTable.close();
    targetTable.delete();
}

Example 8 with MMFMapTable

use of com.ctriposs.sdb.table.MMFMapTable in project sessdb by ppdai.

the class Level1MergerTest method testCase02.

@Test
public void testCase02() throws IOException, ClassNotFoundException {
    int maxSize = AbstractMapTable.INIT_INDEX_ITEMS_PER_TABLE * 4 * 4;
    //int maxSize = 1024;
    MMFMapTable[] sourceTables = new MMFMapTable[3];
    LevelQueue lq1 = new LevelQueue();
    for (int i = 0; i < 3; i++) {
        sourceTables[i] = new MMFMapTable(testDir, SDB.LEVEL1, System.nanoTime() + i, maxSize / 4, 4);
        lq1.addFirst(sourceTables[i]);
    }
    // delete
    MMFMapTable table3 = sourceTables[2];
    int start = 0;
    List<String> keyList = new ArrayList<String>();
    while (start < maxSize) {
        keyList.add(String.valueOf(start));
        start = start + 4;
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (String key : keyList) {
        table3.appendNew(key.getBytes(), Arrays.hashCode(key.getBytes()), key.getBytes(), AbstractMapTable.NO_TIMEOUT, System.currentTimeMillis(), true, false);
    }
    // expiration
    MMFMapTable table2 = sourceTables[1];
    start = 1;
    keyList = new ArrayList<String>();
    while (start < maxSize) {
        keyList.add(String.valueOf(start));
        start = start + 4;
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (String key : keyList) {
        table2.appendNew(key.getBytes(), Arrays.hashCode(key.getBytes()), key.getBytes(), 200, System.currentTimeMillis(), false, false);
    }
    // expire table2
    try {
        Thread.sleep(2000);
    } catch (InterruptedException e) {
    // ignore
    }
    // time to live 60 sec
    MMFMapTable table1 = sourceTables[0];
    start = 2;
    keyList = new ArrayList<String>();
    while (start < maxSize) {
        keyList.add(String.valueOf(start));
        start = start + 4;
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (String key : keyList) {
        table1.appendNew(key.getBytes(), Arrays.hashCode(key.getBytes()), key.getBytes(), 600 * 1000, System.currentTimeMillis(), false, false);
    }
    //int expectedInserts = (int)(table1.getAppendedSize() + table2.getAppendedSize() + table3.getAppendedSize());
    FCMapTable table4 = new FCMapTable(testDir, SDB.LEVEL2, System.nanoTime() + 3, maxSize);
    start = 0;
    keyList = new ArrayList<String>();
    while (start < maxSize) {
        keyList.add(String.valueOf(start));
        start = start + 1;
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (String key : keyList) {
        table4.appendNew(key.getBytes(), Arrays.hashCode(key.getBytes()), key.getBytes(), 1200 * 1000, System.currentTimeMillis(), false, false);
    }
    LevelQueue lq2 = new LevelQueue();
    lq2.add(table4);
    Level1Merger.mergeSort(lq1, lq2, 4, testDir, (short) 2);
    assertTrue(lq1.size() == 0);
    assertTrue(lq2.size() == 1);
    FCMapTable targetTable = (FCMapTable) lq2.poll();
    System.out.println(targetTable.getAppendedSize() + "==" + maxSize / 2);
    assertTrue(targetTable.getAppendedSize() == maxSize / 2);
    /*
		// validate delete
		start = 0;
		while(start < maxSize) {
			GetResult result = targetTable.get(String.valueOf(start).getBytes());
			assertFalse(result.isFound());
			start += 4;
		}
		
		// validate expiration
		start = 1;
		while(start < maxSize) {
			GetResult result = targetTable.get(String.valueOf(start).getBytes());
			assertFalse(result.isFound());
			start += 4;
		}
		
		// validate ttl 60s
		start = 2;
		while(start < maxSize) {
			GetResult result = targetTable.get(String.valueOf(start).getBytes());
			assertTrue(result.isFound());
			assertTrue(result.getTimeToLive() == 600 * 1000);
			start += 4;
		}
		
		// validate ttl 120s
		start = 3;
		while(start < maxSize) {
			GetResult result = targetTable.get(String.valueOf(start).getBytes());
			assertTrue(result.isFound());
			assertTrue(result.getTimeToLive() == 1200 * 1000);
			start += 4;
		}*/
    keyList = new ArrayList<String>();
    for (long i = 0; i < maxSize; i++) {
        if (i % 4 == 0 || i % 4 == 1)
            continue;
        keyList.add(String.valueOf(i));
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    int index = 0;
    for (int i = 0; i < maxSize; i++) {
        // ignore deleted & expired
        if (i % 4 == 0 || i % 4 == 1)
            continue;
        IMapEntry mapEntry = targetTable.getMapEntry(index);
        assertTrue(mapEntry.getIndex() == index);
        assertTrue(new String(mapEntry.getKey()).equals(keyList.get(index)));
        assertTrue(new String(mapEntry.getValue()).equals(keyList.get(index)));
        index++;
    }
    Random random = new Random();
    for (int i = 0; i < 1024; i++) {
        int key = random.nextInt(maxSize);
        // ignore deleted & expired
        if (key % 4 == 0 || key % 4 == 1)
            continue;
        GetResult result = targetTable.get(String.valueOf(key).getBytes());
        assertTrue(result.isFound());
    }
    targetTable.close();
    targetTable.delete();
}

Example 9 with MMFMapTable

use of com.ctriposs.sdb.table.MMFMapTable in project sessdb by ppdai.

the class Level1MergerTest method testCase01.

@Test
public void testCase01() throws IOException, ClassNotFoundException {
    int maxSize = AbstractMapTable.INIT_INDEX_ITEMS_PER_TABLE;
    String value = TestUtil.randomString(1024 * 3);
    MMFMapTable[] sourceTables = new MMFMapTable[4];
    LevelQueue lq1 = new LevelQueue();
    for (int i = 0; i < 4; i++) {
        sourceTables[i] = new MMFMapTable(testDir, SDB.LEVEL1, System.nanoTime() + i, maxSize, 4);
        lq1.addFirst(sourceTables[i]);
    }
    int totalCount = 0;
    for (int i = 0; i < 4; i++) {
        int start = i;
        MMFMapTable table = sourceTables[i];
        List<String> keyList = new ArrayList<String>();
        while (keyList.size() < maxSize) {
            keyList.add(String.valueOf(start));
            totalCount++;
            start = start + 4;
        }
        Collections.sort(keyList, new Comparator<String>() {

            @Override
            public int compare(String arg0, String arg1) {
                int hash0 = Arrays.hashCode(arg0.getBytes());
                int hash1 = Arrays.hashCode(arg1.getBytes());
                if (hash0 < hash1)
                    return -1;
                else if (hash0 > hash1)
                    return 1;
                else
                    return 0;
            }
        });
        for (String key : keyList) {
            table.appendNew(key.getBytes(), value.getBytes(), AbstractMapTable.NO_TIMEOUT);
        }
    }
    LevelQueue lq2 = new LevelQueue();
    long start = System.currentTimeMillis();
    Level1Merger.mergeSort(lq1, lq2, 4, testDir, (short) 3);
    long end = System.currentTimeMillis();
    System.out.println("Time spent to merge " + totalCount + " items in 4 ways  is " + (end - start) / 1000 + "s");
    assertTrue(lq1.size() == 0);
    assertTrue(lq2.size() == 1);
    FCMapTable targetTable = (FCMapTable) lq2.poll();
    assertTrue(targetTable.getLevel() == SDB.LEVEL2);
    assertTrue(targetTable.getAppendedSize() == totalCount);
    assertTrue(totalCount == maxSize * 4);
    List<String> keyList = new ArrayList<String>();
    for (long i = 0; i < totalCount; i++) {
        keyList.add(String.valueOf(i));
    }
    Collections.sort(keyList, new Comparator<String>() {

        @Override
        public int compare(String arg0, String arg1) {
            int hash0 = Arrays.hashCode(arg0.getBytes());
            int hash1 = Arrays.hashCode(arg1.getBytes());
            if (hash0 < hash1)
                return -1;
            else if (hash0 > hash1)
                return 1;
            else
                return 0;
        }
    });
    for (int i = 0; i < totalCount; i++) {
        IMapEntry mapEntry = targetTable.getMapEntry(i);
        assertTrue(mapEntry.getIndex() == i);
        assertTrue(new String(mapEntry.getKey()).equals(keyList.get(i)));
        assertTrue(new String(mapEntry.getValue()).equals(value));
    }
    start = System.currentTimeMillis();
    Random random = new Random();
    for (int i = 0; i < 1024; i++) {
        long key = random.nextInt(totalCount);
        GetResult result = targetTable.get(String.valueOf(key).getBytes());
        assertTrue(result.isFound());
    }
    end = System.currentTimeMillis();
    System.out.println("Time to lookup 1024 random key in the target table is " + (end - start) + "ms");
    targetTable.close();
    targetTable.delete();
}

Example 10 with MMFMapTable

use of com.ctriposs.sdb.table.MMFMapTable in project sessdb by ppdai.

the class MMFMapTableTest method testEmtpy.

@Test
public void testEmtpy() throws IOException, ClassNotFoundException {
    long createdTime = System.nanoTime();
    mapTable = new MMFMapTable(testDir, 1, createdTime, 1000, 4);
    assertTrue(mapTable.getLevel() == 1);
    assertTrue(mapTable.getCreatedTime() == createdTime);
    assertTrue(mapTable.getAppendedSize() == 0);
    assertTrue(mapTable.isEmpty());
    //assertTrue(mapTable.getBackFileSize() == (MMFMapTable.INIT_INDEX_FILE_SIZE + MMFMapTable.INIT_DATA_FILE_SIZE) * Level0Merger.DEFAULT_MERGE_WAYS);
    assertFalse(mapTable.isUsable());
    try {
        @SuppressWarnings("unused") GetResult result = mapTable.get("empty".getBytes());
        fail();
    } catch (IllegalArgumentException iae) {
    }
    try {
        mapTable.getMapEntry(-1);
        fail();
    } catch (IllegalArgumentException iae) {
    }
    try {
        mapTable.getMapEntry(0);
        fail();
    } catch (IllegalArgumentException iae) {
    }
}