Search in sources :

Example 16 with UnsafeRow

use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.

the class UnsafeRowGrouper method addRow.

public void addRow(UnsafeRow rowDataPkg) throws UnsupportedEncodingException {
    UnsafeRow key = getGroupKey(rowDataPkg);
    UnsafeRow value = getValue(rowDataPkg);
    if (aggregationMap.find(key)) {
        UnsafeRow rs = aggregationMap.getAggregationBuffer(key);
        aggregateRow(rs, value);
    } else {
        aggregationMap.put(key, value);
    }
    return;
}
Also used : UnsafeRow(io.mycat.memory.unsafe.row.UnsafeRow)

Example 17 with UnsafeRow

use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.

the class UnsafeFixedWidthAggregationMapSuite method testWithMemoryLeakDetection.

@Test
public void testWithMemoryLeakDetection() throws IOException, NoSuchFieldException, IllegalAccessException {
    MyCatMemory myCatMemory = new MyCatMemory();
    MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager();
    DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
    int fieldCount = 3;
    ColMeta colMeta = null;
    Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("age", colMeta);
    OrderCol[] orderCols = new OrderCol[1];
    OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_DESC);
    orderCols[0] = orderCol;
    groupKeySchema = new StructType(colMetaMap, fieldCount);
    groupKeySchema.setOrderCols(orderCols);
    fieldCount = 3;
    colMeta = null;
    colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_LONGLONG);
    colMetaMap.put("age", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_LONGLONG);
    colMetaMap.put("age1", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    orderCols = new OrderCol[1];
    orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_DESC);
    orderCols[0] = orderCol;
    aggBufferSchema = new StructType(colMetaMap, fieldCount);
    aggBufferSchema.setOrderCols(orderCols);
    /**
   * value
   */
    BufferHolder bufferHolder;
    emptyAggregationBuffer = new UnsafeRow(3);
    bufferHolder = new BufferHolder(emptyAggregationBuffer, 0);
    UnsafeRowWriter unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
    bufferHolder.reset();
    String value = "ok,hello";
    emptyAggregationBuffer.setLong(0, 0);
    emptyAggregationBuffer.setLong(1, 0);
    unsafeRowWriter.write(2, value.getBytes());
    emptyAggregationBuffer.setTotalSize(bufferHolder.totalSize());
    UnsafeFixedWidthAggregationMap map = new UnsafeFixedWidthAggregationMap(emptyAggregationBuffer, aggBufferSchema, groupKeySchema, dataNodeMemoryManager, 2 * 1024, PAGE_SIZE_BYTES, false);
    int i;
    List<UnsafeRow> rows = new ArrayList<UnsafeRow>();
    for (i = 0; i < 1000; i++) {
        String line = "testUnsafeRow" + i;
        /**
     * key
     */
        UnsafeRow groupKey = new UnsafeRow(3);
        bufferHolder = new BufferHolder(groupKey, 0);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
        bufferHolder.reset();
        final byte[] key = getRandomByteArray(rand.nextInt(8));
        String age = "5" + i;
        unsafeRowWriter.write(0, key);
        unsafeRowWriter.write(1, line.getBytes());
        unsafeRowWriter.write(2, age.getBytes());
        groupKey.setTotalSize(bufferHolder.totalSize());
        map.getAggregationBuffer(groupKey);
        rows.add(groupKey);
    }
    Assert.assertEquals(i, rows.size());
    UnsafeRow row = rows.get(12);
    UnsafeRow rs = map.getAggregationBuffer(row);
    rs.setLong(0, 12);
    rs = map.getAggregationBuffer(row);
    Assert.assertEquals(12, rs.getLong(0));
    map.free();
}
Also used : ColMeta(io.mycat.sqlengine.mpp.ColMeta) StructType(io.mycat.memory.unsafe.row.StructType) DataNodeMemoryManager(io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager) UnsafeRowWriter(io.mycat.memory.unsafe.row.UnsafeRowWriter) DataNodeMemoryManager(io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager) MemoryManager(io.mycat.memory.unsafe.memory.mm.MemoryManager) UnsafeRow(io.mycat.memory.unsafe.row.UnsafeRow) BufferHolder(io.mycat.memory.unsafe.row.BufferHolder) OrderCol(io.mycat.sqlengine.mpp.OrderCol) MyCatMemory(io.mycat.memory.MyCatMemory) Test(org.junit.Test)

Aggregations

UnsafeRow (io.mycat.memory.unsafe.row.UnsafeRow)17 BufferHolder (io.mycat.memory.unsafe.row.BufferHolder)10 UnsafeRowWriter (io.mycat.memory.unsafe.row.UnsafeRowWriter)10 StructType (io.mycat.memory.unsafe.row.StructType)6 IOException (java.io.IOException)5 BigDecimal (java.math.BigDecimal)5 DataNodeMemoryManager (io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager)4 ColMeta (io.mycat.sqlengine.mpp.ColMeta)4 OrderCol (io.mycat.sqlengine.mpp.OrderCol)4 MyCatMemory (io.mycat.memory.MyCatMemory)3 UnsafeFixedWidthAggregationMap (io.mycat.memory.unsafe.map.UnsafeFixedWidthAggregationMap)3 MemoryManager (io.mycat.memory.unsafe.memory.mm.MemoryManager)3 Test (org.junit.Test)3 ByteBuffer (java.nio.ByteBuffer)2 MySQLMessage (io.mycat.backend.mysql.MySQLMessage)1 KVIterator (io.mycat.memory.unsafe.KVIterator)1 DataNodeDiskManager (io.mycat.memory.unsafe.storage.DataNodeDiskManager)1 SerializerManager (io.mycat.memory.unsafe.storage.SerializerManager)1 MycatPropertyConf (io.mycat.memory.unsafe.utils.MycatPropertyConf)1 PrefixComparator (io.mycat.memory.unsafe.utils.sort.PrefixComparator)1