Examples with NativeMap org.apache.accumulo.tserver.NativeMap used on opensource projects

Example 6 with NativeMap

use of org.apache.accumulo.tserver.NativeMap in project accumulo by apache.

the class NativeMapIT method test1.

@Test
public void test1() {
    NativeMap nm = new NativeMap();
    Iterator<Entry<Key, Value>> iter = nm.iterator();
    assertFalse(iter.hasNext());
    nm.delete();
}

Example 7 with NativeMap

use of org.apache.accumulo.tserver.NativeMap in project accumulo by apache.

the class NativeMapIT method testConcurrentIter.

@Test
public void testConcurrentIter() throws IOException {
    NativeMap nm = new NativeMap();
    nm.put(newKey(0), newValue(0));
    nm.put(newKey(1), newValue(1));
    nm.put(newKey(3), newValue(3));
    SortedKeyValueIterator<Key, Value> iter = nm.skvIterator();
    // modify map after iter created
    nm.put(newKey(2), newValue(2));
    assertTrue(iter.hasTop());
    assertEquals(iter.getTopKey(), newKey(0));
    iter.next();
    assertTrue(iter.hasTop());
    assertEquals(iter.getTopKey(), newKey(1));
    iter.next();
    assertTrue(iter.hasTop());
    assertEquals(iter.getTopKey(), newKey(2));
    iter.next();
    assertTrue(iter.hasTop());
    assertEquals(iter.getTopKey(), newKey(3));
    iter.next();
    assertFalse(iter.hasTop());
    nm.delete();
}

Example 8 with NativeMap

use of org.apache.accumulo.tserver.NativeMap in project accumulo by apache.

the class NativeMapIT method test11.

@Test
public void test11() {
    NativeMap nm = new NativeMap();
    // insert things with varying field sizes and value sizes
    // generate random data
    Random r = new Random(75);
    ArrayList<Pair<Key, Value>> testData = new ArrayList<>();
    for (int i = 0; i < 100000; i++) {
        Key k = new Key(getRandomBytes(r, 97), getRandomBytes(r, 13), getRandomBytes(r, 31), getRandomBytes(r, 11), (r.nextLong() & 0x7fffffffffffffffl), false, false);
        Value v = new Value(getRandomBytes(r, 511));
        testData.add(new Pair<>(k, v));
    }
    // insert unsorted data
    for (Pair<Key, Value> pair : testData) {
        nm.put(pair.getFirst(), pair.getSecond());
    }
    for (int i = 0; i < 2; i++) {
        // sort data
        Collections.sort(testData, new Comparator<Pair<Key, Value>>() {

            @Override
            public int compare(Pair<Key, Value> o1, Pair<Key, Value> o2) {
                return o1.getFirst().compareTo(o2.getFirst());
            }
        });
        // verify
        Iterator<Entry<Key, Value>> iter1 = nm.iterator();
        Iterator<Pair<Key, Value>> iter2 = testData.iterator();
        while (iter1.hasNext() && iter2.hasNext()) {
            Entry<Key, Value> e = iter1.next();
            Pair<Key, Value> p = iter2.next();
            if (!e.getKey().equals(p.getFirst()))
                throw new RuntimeException("Keys not equal");
            if (!e.getValue().equals(p.getSecond()))
                throw new RuntimeException("Values not equal");
        }
        if (iter1.hasNext())
            throw new RuntimeException("Not all of native map consumed");
        if (iter2.hasNext())
            throw new RuntimeException("Not all of test data consumed");
        System.out.println("test 11 nm mem " + nm.getMemoryUsed());
        // insert data again w/ different value
        Collections.shuffle(testData, r);
        // insert unsorted data
        for (Pair<Key, Value> pair : testData) {
            pair.getSecond().set(getRandomBytes(r, 511));
            nm.put(pair.getFirst(), pair.getSecond());
        }
    }
    nm.delete();
}

Example 9 with NativeMap

use of org.apache.accumulo.tserver.NativeMap in project accumulo by apache.

the class NativeMapConcurrencyTest method create.

static NativeMap create(int numRows, int numCols) {
    NativeMap nm = new NativeMap();
    populate(0, numRows, numCols, nm);
    return nm;
}

Example 10 with NativeMap

use of org.apache.accumulo.tserver.NativeMap in project accumulo by apache.

the class NativeMapPerformanceTest method runPerformanceTest.

static void runPerformanceTest(int numRows, int numCols, int numLookups, String mapType) {
    SortedMap<Key, Value> tm = null;
    NativeMap nm = null;
    if (mapType.equals("SKIP_LIST"))
        tm = new ConcurrentSkipListMap<>();
    else if (mapType.equals("TREE_MAP"))
        tm = Collections.synchronizedSortedMap(new TreeMap<Key, Value>());
    else if (mapType.equals("NATIVE_MAP"))
        nm = new NativeMap();
    else
        throw new IllegalArgumentException(" map type must be SKIP_LIST, TREE_MAP, or NATIVE_MAP");
    Random rand = new Random(19);
    // puts
    long tps = System.currentTimeMillis();
    if (nm != null) {
        for (int i = 0; i < numRows; i++) {
            int row = rand.nextInt(1000000000);
            Mutation m = newMutation(row);
            for (int j = 0; j < numCols; j++) {
                int col = rand.nextInt(1000000);
                Value val = new Value("test".getBytes(UTF_8));
                pc(m, col, val);
            }
            nm.mutate(m, i);
        }
    } else {
        for (int i = 0; i < numRows; i++) {
            int row = rand.nextInt(1000000000);
            for (int j = 0; j < numCols; j++) {
                int col = rand.nextInt(1000000);
                Key key = newKey(row, col);
                Value val = new Value("test".getBytes(UTF_8));
                tm.put(key, val);
            }
        }
    }
    long tpe = System.currentTimeMillis();
    // Iteration
    Iterator<Entry<Key, Value>> iter;
    if (nm != null) {
        iter = nm.iterator();
    } else {
        iter = tm.entrySet().iterator();
    }
    long tis = System.currentTimeMillis();
    while (iter.hasNext()) {
        iter.next();
    }
    long tie = System.currentTimeMillis();
    rand = new Random(19);
    int[] rowsToLookup = new int[numLookups];
    int[] colsToLookup = new int[numLookups];
    for (int i = 0; i < Math.min(numLookups, numRows); i++) {
        int row = rand.nextInt(1000000000);
        int col = -1;
        for (int j = 0; j < numCols; j++) {
            col = rand.nextInt(1000000);
        }
        rowsToLookup[i] = row;
        colsToLookup[i] = col;
    }
    // get
    long tgs = System.currentTimeMillis();
    if (nm != null) {
        for (int i = 0; i < numLookups; i++) {
            Key key = newKey(rowsToLookup[i], colsToLookup[i]);
            if (nm.get(key) == null) {
                throw new RuntimeException("Did not find " + rowsToLookup[i] + " " + colsToLookup[i] + " " + i);
            }
        }
    } else {
        for (int i = 0; i < numLookups; i++) {
            Key key = newKey(rowsToLookup[i], colsToLookup[i]);
            if (tm.get(key) == null) {
                throw new RuntimeException("Did not find " + rowsToLookup[i] + " " + colsToLookup[i] + " " + i);
            }
        }
    }
    long tge = System.currentTimeMillis();
    long memUsed = 0;
    if (nm != null) {
        memUsed = nm.getMemoryUsed();
    }
    int size = (nm == null ? tm.size() : nm.size());
    // delete
    long tds = System.currentTimeMillis();
    if (nm != null)
        nm.delete();
    long tde = System.currentTimeMillis();
    if (tm != null)
        tm.clear();
    System.gc();
    System.gc();
    System.gc();
    System.gc();
    sleepUninterruptibly(3, TimeUnit.SECONDS);
    System.out.printf("mapType:%10s   put rate:%,6.2f  scan rate:%,6.2f  get rate:%,6.2f  delete time : %6.2f  mem : %,d%n", "" + mapType, (numRows * numCols) / ((tpe - tps) / 1000.0), (size) / ((tie - tis) / 1000.0), numLookups / ((tge - tgs) / 1000.0), (tde - tds) / 1000.0, memUsed);
}