Examples with DataNodeMemoryManager io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager used on opensource projects

Example 6 with DataNodeMemoryManager

use of io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager in project Mycat-Server by MyCATApache.

the class UnsafeFixedWidthAggregationMapSuite method testAggregateMap.

@Test
public void testAggregateMap() throws NoSuchFieldException, IllegalAccessException, IOException {
    /**
     * 创造上文环境
     */
    MyCatMemory myCatMemory = new MyCatMemory();
    MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager();
    DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
    /**
       * 构造数据字段group key
       */
    int fieldCount = 2;
    ColMeta colMeta = null;
    Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    OrderCol[] orderCols = new OrderCol[1];
    OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_DESC);
    orderCols[0] = orderCol;
    groupKeySchema = new StructType(colMetaMap, fieldCount);
    groupKeySchema.setOrderCols(orderCols);
    /**
   * 构造数据字段value key
   */
    fieldCount = 4;
    colMeta = null;
    colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_INT);
    colMetaMap.put("age", colMeta);
    colMeta = new ColMeta(3, ColMeta.COL_TYPE_LONGLONG);
    colMetaMap.put("score", colMeta);
    orderCols = new OrderCol[1];
    orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_DESC);
    orderCols[0] = orderCol;
    aggBufferSchema = new StructType(colMetaMap, fieldCount);
    aggBufferSchema.setOrderCols(orderCols);
    /**
     *emtpy Row value
     */
    BufferHolder bufferHolder;
    emptyAggregationBuffer = new UnsafeRow(4);
    bufferHolder = new BufferHolder(emptyAggregationBuffer, 0);
    UnsafeRowWriter unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 4);
    bufferHolder.reset();
    String value = "o";
    unsafeRowWriter.write(0, value.getBytes());
    unsafeRowWriter.write(1, value.getBytes());
    emptyAggregationBuffer.setInt(2, 0);
    emptyAggregationBuffer.setLong(3, 0);
    emptyAggregationBuffer.setTotalSize(bufferHolder.totalSize());
    UnsafeFixedWidthAggregationMap map = new UnsafeFixedWidthAggregationMap(emptyAggregationBuffer, aggBufferSchema, groupKeySchema, dataNodeMemoryManager, 2 * 1024, PAGE_SIZE_BYTES, true);
    /**
       * 造数据
       */
    int i;
    List<UnsafeRow> rows = new ArrayList<UnsafeRow>();
    for (i = 0; i < 100000; i++) {
        /**
       * key
       */
        UnsafeRow groupKey = new UnsafeRow(2);
        bufferHolder = new BufferHolder(groupKey, 0);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 2);
        bufferHolder.reset();
        unsafeRowWriter.write(0, BytesTools.toBytes(rand.nextInt(10000000)));
        unsafeRowWriter.write(1, BytesTools.toBytes(rand.nextInt(10000000)));
        groupKey.setTotalSize(bufferHolder.totalSize());
        UnsafeRow valueKey = new UnsafeRow(4);
        bufferHolder = new BufferHolder(valueKey, 0);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 4);
        bufferHolder.reset();
        unsafeRowWriter.write(0, BytesTools.toBytes(rand.nextInt(10)));
        unsafeRowWriter.write(1, BytesTools.toBytes(rand.nextInt(10)));
        valueKey.setInt(2, i);
        valueKey.setLong(3, 1);
        valueKey.setTotalSize(bufferHolder.totalSize());
        if (map.find(groupKey)) {
            UnsafeRow rs = map.getAggregationBuffer(groupKey);
            rs.setLong(3, i + valueKey.getLong(3));
            rs.setInt(2, 100 + valueKey.getInt(2));
        } else {
            map.put(groupKey, valueKey);
        }
        rows.add(valueKey);
    }
    KVIterator<UnsafeRow, UnsafeRow> iter = map.iterator();
    int j = 0;
    while (iter.next()) {
        Assert.assertEquals(j, iter.getValue().getInt(2));
        j++;
        iter.getValue().setInt(2, 5000000);
        iter.getValue().setLong(3, 600000);
    }
    Assert.assertEquals(rows.size(), j);
    int k = 0;
    KVIterator<UnsafeRow, UnsafeRow> iter1 = map.iterator();
    while (iter1.next()) {
        k++;
        // LOGGER.error("(" + BytesTools.toInt(iter1.getKey().getBinary(0)) + "," +
        //       iter1.getValue().getInt(2) +"," +iter1.getValue().getLong(3)+")");
        Assert.assertEquals(5000000, iter1.getValue().getInt(2));
        Assert.assertEquals(600000, iter1.getValue().getLong(3));
    }
    Assert.assertEquals(j, k);
    map.free();
}

Example 7 with DataNodeMemoryManager

use of io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager in project Mycat-Server by MyCATApache.

the class TaskMemoryManagerSuite method encodePageNumberAndOffsetOffHeap.

@Test
public void encodePageNumberAndOffsetOffHeap() {
    final MycatPropertyConf conf = new MycatPropertyConf().set("mycat.memory.offHeap.enabled", "true").set("mycat.memory.offHeap.size", "1000");
    final DataNodeMemoryManager manager = new DataNodeMemoryManager(new TestMemoryManager(conf), 0);
    final MemoryBlock dataPage = manager.allocatePage(256, null);
    // In off-heap mode, an offset is an absolute address that may require more than 51 bits to
    // encode. This map exercises that corner-case:
    final long offset = ((1L << DataNodeMemoryManager.OFFSET_BITS) + 10);
    final long encodedAddress = manager.encodePageNumberAndOffset(dataPage, offset);
    Assert.assertEquals(null, manager.getPage(encodedAddress));
    Assert.assertEquals(offset, manager.getOffsetInPage(encodedAddress));
}

Example 8 with DataNodeMemoryManager

use of io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager in project Mycat-Server by MyCATApache.

the class TaskMemoryManagerSuite method encodePageNumberAndOffsetOnHeap.

@Test
public void encodePageNumberAndOffsetOnHeap() {
    final DataNodeMemoryManager manager = new DataNodeMemoryManager(new TestMemoryManager(new MycatPropertyConf().set("mycat.memory.offHeap.enabled", "false")), 0);
    final MemoryBlock dataPage = manager.allocatePage(256, null);
    final long encodedAddress = manager.encodePageNumberAndOffset(dataPage, 64);
    Assert.assertEquals(dataPage.getBaseObject(), manager.getPage(encodedAddress));
    Assert.assertEquals(64, manager.getOffsetInPage(encodedAddress));
}

Example 9 with DataNodeMemoryManager

use of io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager in project Mycat-Server by MyCATApache.

the class UnsafeFixedWidthAggregationMapSuite method testWithMemoryLeakDetection.

@Test
public void testWithMemoryLeakDetection() throws IOException, NoSuchFieldException, IllegalAccessException {
    MyCatMemory myCatMemory = new MyCatMemory();
    MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager();
    DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
    int fieldCount = 3;
    ColMeta colMeta = null;
    Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("age", colMeta);
    OrderCol[] orderCols = new OrderCol[1];
    OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_DESC);
    orderCols[0] = orderCol;
    groupKeySchema = new StructType(colMetaMap, fieldCount);
    groupKeySchema.setOrderCols(orderCols);
    fieldCount = 3;
    colMeta = null;
    colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_LONGLONG);
    colMetaMap.put("age", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_LONGLONG);
    colMetaMap.put("age1", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    orderCols = new OrderCol[1];
    orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_DESC);
    orderCols[0] = orderCol;
    aggBufferSchema = new StructType(colMetaMap, fieldCount);
    aggBufferSchema.setOrderCols(orderCols);
    /**
   * value
   */
    BufferHolder bufferHolder;
    emptyAggregationBuffer = new UnsafeRow(3);
    bufferHolder = new BufferHolder(emptyAggregationBuffer, 0);
    UnsafeRowWriter unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
    bufferHolder.reset();
    String value = "ok,hello";
    emptyAggregationBuffer.setLong(0, 0);
    emptyAggregationBuffer.setLong(1, 0);
    unsafeRowWriter.write(2, value.getBytes());
    emptyAggregationBuffer.setTotalSize(bufferHolder.totalSize());
    UnsafeFixedWidthAggregationMap map = new UnsafeFixedWidthAggregationMap(emptyAggregationBuffer, aggBufferSchema, groupKeySchema, dataNodeMemoryManager, 2 * 1024, PAGE_SIZE_BYTES, false);
    int i;
    List<UnsafeRow> rows = new ArrayList<UnsafeRow>();
    for (i = 0; i < 1000; i++) {
        String line = "testUnsafeRow" + i;
        /**
     * key
     */
        UnsafeRow groupKey = new UnsafeRow(3);
        bufferHolder = new BufferHolder(groupKey, 0);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
        bufferHolder.reset();
        final byte[] key = getRandomByteArray(rand.nextInt(8));
        String age = "5" + i;
        unsafeRowWriter.write(0, key);
        unsafeRowWriter.write(1, line.getBytes());
        unsafeRowWriter.write(2, age.getBytes());
        groupKey.setTotalSize(bufferHolder.totalSize());
        map.getAggregationBuffer(groupKey);
        rows.add(groupKey);
    }
    Assert.assertEquals(i, rows.size());
    UnsafeRow row = rows.get(12);
    UnsafeRow rs = map.getAggregationBuffer(row);
    rs.setLong(0, 12);
    rs = map.getAggregationBuffer(row);
    Assert.assertEquals(12, rs.getLong(0));
    map.free();
}

Example 10 with DataNodeMemoryManager

use of io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager in project Mycat-Server by MyCATApache.

the class TaskMemoryManagerSuite method leakedPageMemoryIsDetected.

@Test
public void leakedPageMemoryIsDetected() {
    final DataNodeMemoryManager manager = new DataNodeMemoryManager(new ResultMergeMemoryManager(new MycatPropertyConf().set("mycat.memory.offHeap.enabled", "false").set("mycat.memory.offHeap.size", "32768"), 1, Long.MAX_VALUE), 0);
    // leak memory
    manager.allocatePage(4096, null);
    Assert.assertEquals(4096, manager.getMemoryConsumptionForThisConnection());
    Assert.assertEquals(4096, manager.cleanUpAllAllocatedMemory());
}