Examples with StructType io.mycat.memory.unsafe.row.StructType used on opensource projects

Example 1 with StructType

use of io.mycat.memory.unsafe.row.StructType in project Mycat_plus by coderczp.

the class UnsafeFixedWidthAggregationMapSuite method testAggregateMap.

@Test
public void testAggregateMap() throws NoSuchFieldException, IllegalAccessException, IOException {
    /**
     * 创造上文环境
     */
    MyCatMemory myCatMemory = new MyCatMemory();
    MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager();
    DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
    /**
     * 构造数据字段group key
     */
    int fieldCount = 2;
    ColMeta colMeta = null;
    Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    OrderCol[] orderCols = new OrderCol[1];
    OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_DESC);
    orderCols[0] = orderCol;
    groupKeySchema = new StructType(colMetaMap, fieldCount);
    groupKeySchema.setOrderCols(orderCols);
    /**
     * 构造数据字段value key
     */
    fieldCount = 4;
    colMeta = null;
    colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_INT);
    colMetaMap.put("age", colMeta);
    colMeta = new ColMeta(3, ColMeta.COL_TYPE_LONGLONG);
    colMetaMap.put("score", colMeta);
    orderCols = new OrderCol[1];
    orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_DESC);
    orderCols[0] = orderCol;
    aggBufferSchema = new StructType(colMetaMap, fieldCount);
    aggBufferSchema.setOrderCols(orderCols);
    /**
     *emtpy Row value
     */
    BufferHolder bufferHolder;
    emptyAggregationBuffer = new UnsafeRow(4);
    bufferHolder = new BufferHolder(emptyAggregationBuffer, 0);
    UnsafeRowWriter unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 4);
    bufferHolder.reset();
    String value = "o";
    unsafeRowWriter.write(0, value.getBytes());
    unsafeRowWriter.write(1, value.getBytes());
    emptyAggregationBuffer.setInt(2, 0);
    emptyAggregationBuffer.setLong(3, 0);
    emptyAggregationBuffer.setTotalSize(bufferHolder.totalSize());
    UnsafeFixedWidthAggregationMap map = new UnsafeFixedWidthAggregationMap(emptyAggregationBuffer, aggBufferSchema, groupKeySchema, dataNodeMemoryManager, 2 * 1024, PAGE_SIZE_BYTES, true);
    /**
     * 造数据
     */
    int i;
    List<UnsafeRow> rows = new ArrayList<UnsafeRow>();
    for (i = 0; i < 100000; i++) {
        /**
         * key
         */
        UnsafeRow groupKey = new UnsafeRow(2);
        bufferHolder = new BufferHolder(groupKey, 0);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 2);
        bufferHolder.reset();
        unsafeRowWriter.write(0, BytesTools.toBytes(rand.nextInt(10000000)));
        unsafeRowWriter.write(1, BytesTools.toBytes(rand.nextInt(10000000)));
        groupKey.setTotalSize(bufferHolder.totalSize());
        UnsafeRow valueKey = new UnsafeRow(4);
        bufferHolder = new BufferHolder(valueKey, 0);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 4);
        bufferHolder.reset();
        unsafeRowWriter.write(0, BytesTools.toBytes(rand.nextInt(10)));
        unsafeRowWriter.write(1, BytesTools.toBytes(rand.nextInt(10)));
        valueKey.setInt(2, i);
        valueKey.setLong(3, 1);
        valueKey.setTotalSize(bufferHolder.totalSize());
        if (map.find(groupKey)) {
            UnsafeRow rs = map.getAggregationBuffer(groupKey);
            rs.setLong(3, i + valueKey.getLong(3));
            rs.setInt(2, 100 + valueKey.getInt(2));
        } else {
            map.put(groupKey, valueKey);
        }
        rows.add(valueKey);
    }
    KVIterator<UnsafeRow, UnsafeRow> iter = map.iterator();
    int j = 0;
    while (iter.next()) {
        Assert.assertEquals(j, iter.getValue().getInt(2));
        j++;
        iter.getValue().setInt(2, 5000000);
        iter.getValue().setLong(3, 600000);
    }
    Assert.assertEquals(rows.size(), j);
    int k = 0;
    KVIterator<UnsafeRow, UnsafeRow> iter1 = map.iterator();
    while (iter1.next()) {
        k++;
        // LOGGER.error("(" + BytesTools.toInt(iter1.getKey().getBinary(0)) + "," +
        // iter1.getValue().getInt(2) +"," +iter1.getValue().getLong(3)+")");
        Assert.assertEquals(5000000, iter1.getValue().getInt(2));
        Assert.assertEquals(600000, iter1.getValue().getLong(3));
    }
    Assert.assertEquals(j, k);
    map.free();
}

Example 2 with StructType

use of io.mycat.memory.unsafe.row.StructType in project Mycat_plus by coderczp.

the class UnsafeFixedWidthAggregationMapSuite method testWithMemoryLeakDetection.

@Test
public void testWithMemoryLeakDetection() throws IOException, NoSuchFieldException, IllegalAccessException {
    MyCatMemory myCatMemory = new MyCatMemory();
    MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager();
    DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
    int fieldCount = 3;
    ColMeta colMeta = null;
    Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("age", colMeta);
    OrderCol[] orderCols = new OrderCol[1];
    OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_DESC);
    orderCols[0] = orderCol;
    groupKeySchema = new StructType(colMetaMap, fieldCount);
    groupKeySchema.setOrderCols(orderCols);
    fieldCount = 3;
    colMeta = null;
    colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_LONGLONG);
    colMetaMap.put("age", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_LONGLONG);
    colMetaMap.put("age1", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    orderCols = new OrderCol[1];
    orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_DESC);
    orderCols[0] = orderCol;
    aggBufferSchema = new StructType(colMetaMap, fieldCount);
    aggBufferSchema.setOrderCols(orderCols);
    /**
     * value
     */
    BufferHolder bufferHolder;
    emptyAggregationBuffer = new UnsafeRow(3);
    bufferHolder = new BufferHolder(emptyAggregationBuffer, 0);
    UnsafeRowWriter unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
    bufferHolder.reset();
    String value = "ok,hello";
    emptyAggregationBuffer.setLong(0, 0);
    emptyAggregationBuffer.setLong(1, 0);
    unsafeRowWriter.write(2, value.getBytes());
    emptyAggregationBuffer.setTotalSize(bufferHolder.totalSize());
    UnsafeFixedWidthAggregationMap map = new UnsafeFixedWidthAggregationMap(emptyAggregationBuffer, aggBufferSchema, groupKeySchema, dataNodeMemoryManager, 2 * 1024, PAGE_SIZE_BYTES, false);
    int i;
    List<UnsafeRow> rows = new ArrayList<UnsafeRow>();
    for (i = 0; i < 1000; i++) {
        String line = "testUnsafeRow" + i;
        /**
         * key
         */
        UnsafeRow groupKey = new UnsafeRow(3);
        bufferHolder = new BufferHolder(groupKey, 0);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
        bufferHolder.reset();
        final byte[] key = getRandomByteArray(rand.nextInt(8));
        String age = "5" + i;
        unsafeRowWriter.write(0, key);
        unsafeRowWriter.write(1, line.getBytes());
        unsafeRowWriter.write(2, age.getBytes());
        groupKey.setTotalSize(bufferHolder.totalSize());
        map.getAggregationBuffer(groupKey);
        rows.add(groupKey);
    }
    Assert.assertEquals(i, rows.size());
    UnsafeRow row = rows.get(12);
    UnsafeRow rs = map.getAggregationBuffer(row);
    rs.setLong(0, 12);
    rs = map.getAggregationBuffer(row);
    Assert.assertEquals(12, rs.getLong(0));
    map.free();
}

Example 3 with StructType

use of io.mycat.memory.unsafe.row.StructType in project Mycat_plus by coderczp.

the class UnsafeExternalRowSorterTest method testUnsafeExternalRowSorter.

/**
 * 测试类型 LONG，INT，SHORT,Float，Double，String，Binary
 * 经测试基数排序可以适用上述数据类型，大大提高排序速度
 */
@Test
public void testUnsafeExternalRowSorter() throws NoSuchFieldException, IllegalAccessException, IOException {
    MyCatMemory myCatMemory = new MyCatMemory();
    MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager();
    DataNodeDiskManager blockManager = myCatMemory.getBlockManager();
    SerializerManager serializerManager = myCatMemory.getSerializerManager();
    MycatPropertyConf conf = myCatMemory.getConf();
    DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
    /**
     * 1.schema ,模拟一个field字段值
     */
    int fieldCount = 3;
    ColMeta colMeta = null;
    Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("age", colMeta);
    OrderCol[] orderCols = new OrderCol[1];
    OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_ASC);
    orderCols[0] = orderCol;
    /**
     * 2 .PrefixComputer
     */
    StructType schema = new StructType(colMetaMap, fieldCount);
    schema.setOrderCols(orderCols);
    UnsafeExternalRowSorter.PrefixComputer prefixComputer = new RowPrefixComputer(schema);
    /**
     * 3 .PrefixComparator 默认是ASC，可以选择DESC
     */
    final PrefixComparator prefixComparator = PrefixComparators.LONG;
    UnsafeExternalRowSorter sorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "1m"), true, /**
     *使用基数排序？true or false
     */
    true);
    UnsafeRow unsafeRow;
    BufferHolder bufferHolder;
    UnsafeRowWriter unsafeRowWriter;
    String line = "testUnsafeRow";
    // List<Float> floats = new ArrayList<Float>();
    List<Long> longs = new ArrayList<Long>();
    final Random rand = new Random(42);
    for (int i = 0; i < TEST_SIZE; i++) {
        unsafeRow = new UnsafeRow(3);
        bufferHolder = new BufferHolder(unsafeRow);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
        bufferHolder.reset();
        String key = getRandomString(rand.nextInt(300) + 100);
        // long v = rand.nextLong();
        // longs.add(v);
        unsafeRowWriter.write(0, key.getBytes());
        // unsafeRowWriter.write(0, BytesTools.toBytes(v));
        unsafeRowWriter.write(1, line.getBytes());
        unsafeRowWriter.write(2, ("35" + 1).getBytes());
        unsafeRow.setTotalSize(bufferHolder.totalSize());
        sorter.insertRow(unsafeRow);
    }
    Iterator<UnsafeRow> iter = sorter.sort();
    /*
         float [] com = new float[floats.size()];
        for (int i = 0; i <floats.size() ; i++) {
                com[i] = floats.get(i);
        }
        Arrays.sort(com);


        long[] com = new long[longs.size()];
        for (int i = 0; i < longs.size() ; i++) {
            com[i] = longs.get(i);
        }

        Arrays.sort(com);
        */
    UnsafeRow row = null;
    int indexprint = 0;
    while (iter.hasNext()) {
        row = iter.next();
        // LOGGER.error(indexprint + "    " +  row.getUTF8String(0));
        // Assert.assertEquals(com[indexprint],
        // BytesTools.toLong(row.getBinary(0)));
        // Double c = Double.parseDouble(String.valueOf(com[indexprint])) ;
        // Double c1 = Double.parseDouble(String.valueOf(BytesTools.toFloat(row.getBinary(0)))) ;
        // Assert.assertEquals(0,c.compareTo(c1));
        indexprint++;
    }
    Assert.assertEquals(TEST_SIZE, indexprint);
}

Example 4 with StructType

use of io.mycat.memory.unsafe.row.StructType in project Mycat_plus by coderczp.

the class UnsafeRowGrouper method initEmptyValueKey.

private void initEmptyValueKey() {
    /**
     * 构造valuerow
     */
    emptyAggregationBuffer = new UnsafeRow(this.valuefieldCount);
    bufferHolder = new BufferHolder(emptyAggregationBuffer, 0);
    unsafeRowWriter = new UnsafeRowWriter(bufferHolder, this.valuefieldCount);
    bufferHolder.reset();
    ColMeta curColMeta = null;
    for (Map.Entry<String, ColMeta> fieldEntry : columToIndx.entrySet()) {
        curColMeta = fieldEntry.getValue();
        switch(curColMeta.colType) {
            case ColMeta.COL_TYPE_BIT:
                emptyAggregationBuffer.setByte(curColMeta.colIndex, (byte) 0);
                break;
            case ColMeta.COL_TYPE_INT:
            case ColMeta.COL_TYPE_INT24:
            case ColMeta.COL_TYPE_LONG:
                emptyAggregationBuffer.setInt(curColMeta.colIndex, 0);
                break;
            case ColMeta.COL_TYPE_SHORT:
                emptyAggregationBuffer.setShort(curColMeta.colIndex, (short) 0);
                break;
            case ColMeta.COL_TYPE_LONGLONG:
                emptyAggregationBuffer.setLong(curColMeta.colIndex, 0);
                break;
            case ColMeta.COL_TYPE_FLOAT:
                emptyAggregationBuffer.setFloat(curColMeta.colIndex, 0);
                break;
            case ColMeta.COL_TYPE_DOUBLE:
                emptyAggregationBuffer.setDouble(curColMeta.colIndex, 0);
                break;
            case ColMeta.COL_TYPE_NEWDECIMAL:
                // emptyAggregationBuffer.setDouble(curColMeta.colIndex, 0);
                unsafeRowWriter.write(curColMeta.colIndex, new BigDecimal(0L));
                break;
            default:
                unsafeRowWriter.write(curColMeta.colIndex, "init".getBytes());
                break;
        }
    }
    emptyAggregationBuffer.setTotalSize(bufferHolder.totalSize());
    aggBufferSchema = new StructType(columToIndx, this.valuefieldCount);
    aggBufferSchema.setOrderCols(null);
}

Example 5 with StructType

use of io.mycat.memory.unsafe.row.StructType in project Mycat-Server by MyCATApache.

the class UnsafeExternalRowSorterTest method testUnsafeExternalRowSorter.

/**
 * 测试类型 LONG，INT，SHORT,Float，Double，String，Binary
 * 经测试基数排序可以适用上述数据类型，大大提高排序速度
 */
@Test
public void testUnsafeExternalRowSorter() throws NoSuchFieldException, IllegalAccessException, IOException {
    MyCatMemory myCatMemory = new MyCatMemory();
    MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager();
    DataNodeDiskManager blockManager = myCatMemory.getBlockManager();
    SerializerManager serializerManager = myCatMemory.getSerializerManager();
    MycatPropertyConf conf = myCatMemory.getConf();
    DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
    /**
     * 1.schema ,模拟一个field字段值
     */
    int fieldCount = 3;
    ColMeta colMeta = null;
    Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("age", colMeta);
    OrderCol[] orderCols = new OrderCol[1];
    OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_ASC);
    orderCols[0] = orderCol;
    /**
     * 2 .PrefixComputer
     */
    StructType schema = new StructType(colMetaMap, fieldCount);
    schema.setOrderCols(orderCols);
    UnsafeExternalRowSorter.PrefixComputer prefixComputer = new RowPrefixComputer(schema);
    /**
     * 3 .PrefixComparator 默认是ASC，可以选择DESC
     */
    final PrefixComparator prefixComparator = PrefixComparators.LONG;
    UnsafeExternalRowSorter sorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "1m"), true, /**
     *使用基数排序？true or false
     */
    true);
    UnsafeRow unsafeRow;
    BufferHolder bufferHolder;
    UnsafeRowWriter unsafeRowWriter;
    String line = "testUnsafeRow";
    // List<Float> floats = new ArrayList<Float>();
    List<Long> longs = new ArrayList<Long>();
    final Random rand = new Random(42);
    for (int i = 0; i < TEST_SIZE; i++) {
        unsafeRow = new UnsafeRow(3);
        bufferHolder = new BufferHolder(unsafeRow);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
        bufferHolder.reset();
        String key = getRandomString(rand.nextInt(300) + 100);
        // long v = rand.nextLong();
        // longs.add(v);
        unsafeRowWriter.write(0, key.getBytes());
        // unsafeRowWriter.write(0, BytesTools.toBytes(v));
        unsafeRowWriter.write(1, line.getBytes());
        unsafeRowWriter.write(2, ("35" + 1).getBytes());
        unsafeRow.setTotalSize(bufferHolder.totalSize());
        sorter.insertRow(unsafeRow);
    }
    Iterator<UnsafeRow> iter = sorter.sort();
    /*
         float [] com = new float[floats.size()];
        for (int i = 0; i <floats.size() ; i++) {
                com[i] = floats.get(i);
        }
        Arrays.sort(com);


        long[] com = new long[longs.size()];
        for (int i = 0; i < longs.size() ; i++) {
            com[i] = longs.get(i);
        }

        Arrays.sort(com);
        */
    UnsafeRow row = null;
    int indexprint = 0;
    while (iter.hasNext()) {
        row = iter.next();
        // LOGGER.error(indexprint + "    " +  row.getUTF8String(0));
        // Assert.assertEquals(com[indexprint],
        // BytesTools.toLong(row.getBinary(0)));
        // Double c = Double.parseDouble(String.valueOf(com[indexprint])) ;
        // Double c1 = Double.parseDouble(String.valueOf(BytesTools.toFloat(row.getBinary(0)))) ;
        // Assert.assertEquals(0,c.compareTo(c1));
        indexprint++;
    }
    Assert.assertEquals(TEST_SIZE, indexprint);
}