Examples with UnsafeExternalRowSorter io.mycat.memory.unsafe.utils.sort.UnsafeExternalRowSorter used on opensource projects

Search in sources :

Example 1 with UnsafeExternalRowSorter

use of io.mycat.memory.unsafe.utils.sort.UnsafeExternalRowSorter in project Mycat-Server by MyCATApache.

the class DataNodeMergeManager method onRowMetaData.

public void onRowMetaData(Map<String, ColMeta> columToIndx, int fieldCount) throws IOException {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("field metadata keys:" + columToIndx.keySet());
        LOGGER.debug("field metadata values:" + columToIndx.values());
    }
    OrderCol[] orderCols = null;
    StructType schema = null;
    UnsafeExternalRowSorter.PrefixComputer prefixComputer = null;
    PrefixComparator prefixComparator = null;
    DataNodeMemoryManager dataNodeMemoryManager = null;
    UnsafeExternalRowSorter sorter = null;
    int[] groupColumnIndexs = null;
    this.fieldCount = fieldCount;
    if (rrs.getGroupByCols() != null) {
        groupColumnIndexs = toColumnIndex(rrs.getGroupByCols(), columToIndx);
        if (LOGGER.isDebugEnabled()) {
            for (int i = 0; i < rrs.getGroupByCols().length; i++) {
                LOGGER.debug("groupColumnIndexs:" + rrs.getGroupByCols()[i]);
            }
        }
    }
    if (rrs.getHavingCols() != null) {
        ColMeta colMeta = columToIndx.get(rrs.getHavingCols().getLeft().toUpperCase());
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("getHavingCols:" + rrs.getHavingCols().toString());
        }
        if (colMeta != null) {
            rrs.getHavingCols().setColMeta(colMeta);
        }
    }
    if (rrs.isHasAggrColumn()) {
        List<MergeCol> mergCols = new LinkedList<MergeCol>();
        Map<String, Integer> mergeColsMap = rrs.getMergeCols();
        if (mergeColsMap != null) {
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("isHasAggrColumn:" + mergeColsMap.toString());
            }
            for (Map.Entry<String, Integer> mergEntry : mergeColsMap.entrySet()) {
                String colName = mergEntry.getKey().toUpperCase();
                int type = mergEntry.getValue();
                if (MergeCol.MERGE_AVG == type) {
                    ColMeta sumColMeta = columToIndx.get(colName + "SUM");
                    ColMeta countColMeta = columToIndx.get(colName + "COUNT");
                    if (sumColMeta != null && countColMeta != null) {
                        ColMeta colMeta = new ColMeta(sumColMeta.colIndex, countColMeta.colIndex, sumColMeta.getColType());
                        mergCols.add(new MergeCol(colMeta, mergEntry.getValue()));
                    }
                } else {
                    ColMeta colMeta = columToIndx.get(colName);
                    mergCols.add(new MergeCol(colMeta, mergEntry.getValue()));
                }
            }
        }
        // add no alias merg column
        for (Map.Entry<String, ColMeta> fieldEntry : columToIndx.entrySet()) {
            String colName = fieldEntry.getKey();
            int result = MergeCol.tryParseAggCol(colName);
            if (result != MergeCol.MERGE_UNSUPPORT && result != MergeCol.MERGE_NOMERGE) {
                mergCols.add(new MergeCol(fieldEntry.getValue(), result));
            }
        }
        /**
             * Group操作
             */
        unsafeRowGrouper = new UnsafeRowGrouper(columToIndx, rrs.getGroupByCols(), mergCols.toArray(new MergeCol[mergCols.size()]), rrs.getHavingCols());
    }
    if (rrs.getOrderByCols() != null) {
        LinkedHashMap<String, Integer> orders = rrs.getOrderByCols();
        orderCols = new OrderCol[orders.size()];
        int i = 0;
        for (Map.Entry<String, Integer> entry : orders.entrySet()) {
            String key = StringUtil.removeBackquote(entry.getKey().toUpperCase());
            ColMeta colMeta = columToIndx.get(key);
            if (colMeta == null) {
                throw new IllegalArgumentException("all columns in order by clause should be in the selected column list!" + entry.getKey());
            }
            orderCols[i++] = new OrderCol(colMeta, entry.getValue());
        }
        /**
             * 构造全局排序器
             */
        schema = new StructType(columToIndx, fieldCount);
        schema.setOrderCols(orderCols);
        prefixComputer = new RowPrefixComputer(schema);
        //            if(orderCols.length>0
        //                    && orderCols[0].getOrderType()
        //                    == OrderCol.COL_ORDER_TYPE_ASC){
        //                prefixComparator = PrefixComparators.LONG;
        //            }else {
        //                prefixComparator = PrefixComparators.LONG_DESC;
        //            }
        prefixComparator = getPrefixComparator(orderCols);
        dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
        /**
             * 默认排序，只是将数据连续存储到内存中即可。
             */
        globalSorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "1m"), false, /**是否使用基数排序*/
        true);
    }
    if (conf.getBoolean("mycat.stream.output.result", false) && globalSorter == null && unsafeRowGrouper == null) {
        setStreamOutputResult(true);
    } else {
        /**
             * 1.schema 
             */
        schema = new StructType(columToIndx, fieldCount);
        schema.setOrderCols(orderCols);
        /**
             * 2 .PrefixComputer
             */
        prefixComputer = new RowPrefixComputer(schema);
        /**
             * 3 .PrefixComparator 默认是ASC，可以选择DESC
             */
        prefixComparator = PrefixComparators.LONG;
        dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
        globalMergeResult = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "1m"), false, /**是否使用基数排序*/
        false);
    }
}
Also used : PrefixComparator(io.mycat.memory.unsafe.utils.sort.PrefixComparator) UnsafeExternalRowSorter(io.mycat.memory.unsafe.utils.sort.UnsafeExternalRowSorter) StructType(io.mycat.memory.unsafe.row.StructType) DataNodeMemoryManager(io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager) RowPrefixComputer(io.mycat.memory.unsafe.utils.sort.RowPrefixComputer) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap)

Example 2 with UnsafeExternalRowSorter

use of io.mycat.memory.unsafe.utils.sort.UnsafeExternalRowSorter in project Mycat-Server by MyCATApache.

the class UnsafeExternalRowSorterTest method testUnsafeExternalRowSorter.

/**
     * 测试类型 LONG，INT，SHORT,Float，Double，String，Binary
     * 经测试基数排序可以适用上述数据类型，大大提高排序速度

     */
@Test
public void testUnsafeExternalRowSorter() throws NoSuchFieldException, IllegalAccessException, IOException {
    MyCatMemory myCatMemory = new MyCatMemory();
    MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager();
    DataNodeDiskManager blockManager = myCatMemory.getBlockManager();
    SerializerManager serializerManager = myCatMemory.getSerializerManager();
    MycatPropertyConf conf = myCatMemory.getConf();
    DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
    /**
         * 1.schema ,模拟一个field字段值
         *
         */
    int fieldCount = 3;
    ColMeta colMeta = null;
    Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("age", colMeta);
    OrderCol[] orderCols = new OrderCol[1];
    OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_ASC);
    orderCols[0] = orderCol;
    /**
         * 2 .PrefixComputer
         */
    StructType schema = new StructType(colMetaMap, fieldCount);
    schema.setOrderCols(orderCols);
    UnsafeExternalRowSorter.PrefixComputer prefixComputer = new RowPrefixComputer(schema);
    /**
         * 3 .PrefixComparator 默认是ASC，可以选择DESC
         */
    final PrefixComparator prefixComparator = PrefixComparators.LONG;
    UnsafeExternalRowSorter sorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "1m"), true, /**使用基数排序？true or false*/
    true);
    UnsafeRow unsafeRow;
    BufferHolder bufferHolder;
    UnsafeRowWriter unsafeRowWriter;
    String line = "testUnsafeRow";
    // List<Float> floats = new ArrayList<Float>();
    List<Long> longs = new ArrayList<Long>();
    final Random rand = new Random(42);
    for (int i = 0; i < TEST_SIZE; i++) {
        unsafeRow = new UnsafeRow(3);
        bufferHolder = new BufferHolder(unsafeRow);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
        bufferHolder.reset();
        String key = getRandomString(rand.nextInt(300) + 100);
        //long v = rand.nextLong();
        // longs.add(v);
        unsafeRowWriter.write(0, key.getBytes());
        // unsafeRowWriter.write(0, BytesTools.toBytes(v));
        unsafeRowWriter.write(1, line.getBytes());
        unsafeRowWriter.write(2, ("35" + 1).getBytes());
        unsafeRow.setTotalSize(bufferHolder.totalSize());
        sorter.insertRow(unsafeRow);
    }
    Iterator<UnsafeRow> iter = sorter.sort();
    /*
         float [] com = new float[floats.size()];
        for (int i = 0; i <floats.size() ; i++) {
                com[i] = floats.get(i);
        }
        Arrays.sort(com);


        long[] com = new long[longs.size()];
        for (int i = 0; i < longs.size() ; i++) {
            com[i] = longs.get(i);
        }

        Arrays.sort(com);
        */
    UnsafeRow row = null;
    int indexprint = 0;
    while (iter.hasNext()) {
        row = iter.next();
        // LOGGER.error(indexprint + "    " +  row.getUTF8String(0));
        //Assert.assertEquals(com[indexprint],
        //        BytesTools.toLong(row.getBinary(0)));
        // Double c = Double.parseDouble(String.valueOf(com[indexprint])) ;
        // Double c1 = Double.parseDouble(String.valueOf(BytesTools.toFloat(row.getBinary(0)))) ;
        //  Assert.assertEquals(0,c.compareTo(c1));
        indexprint++;
    }
    Assert.assertEquals(TEST_SIZE, indexprint);
}
Also used : UnsafeExternalRowSorter(io.mycat.memory.unsafe.utils.sort.UnsafeExternalRowSorter) ColMeta(io.mycat.sqlengine.mpp.ColMeta) StructType(io.mycat.memory.unsafe.row.StructType) DataNodeDiskManager(io.mycat.memory.unsafe.storage.DataNodeDiskManager) MycatPropertyConf(io.mycat.memory.unsafe.utils.MycatPropertyConf) UnsafeRow(io.mycat.memory.unsafe.row.UnsafeRow) BufferHolder(io.mycat.memory.unsafe.row.BufferHolder) OrderCol(io.mycat.sqlengine.mpp.OrderCol) PrefixComparator(io.mycat.memory.unsafe.utils.sort.PrefixComparator) SerializerManager(io.mycat.memory.unsafe.storage.SerializerManager) DataNodeMemoryManager(io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager) UnsafeRowWriter(io.mycat.memory.unsafe.row.UnsafeRowWriter) DataNodeMemoryManager(io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager) MemoryManager(io.mycat.memory.unsafe.memory.mm.MemoryManager) RowPrefixComputer(io.mycat.memory.unsafe.utils.sort.RowPrefixComputer) MyCatMemory(io.mycat.memory.MyCatMemory) Test(org.junit.Test)

Example 3 with UnsafeExternalRowSorter

use of io.mycat.memory.unsafe.utils.sort.UnsafeExternalRowSorter in project Mycat_plus by coderczp.

the class UnsafeExternalRowSorterTest method testUnsafeExternalRowSorter.

/**
 * 测试类型 LONG，INT，SHORT,Float，Double，String，Binary
 * 经测试基数排序可以适用上述数据类型，大大提高排序速度
 */
@Test
public void testUnsafeExternalRowSorter() throws NoSuchFieldException, IllegalAccessException, IOException {
    MyCatMemory myCatMemory = new MyCatMemory();
    MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager();
    DataNodeDiskManager blockManager = myCatMemory.getBlockManager();
    SerializerManager serializerManager = myCatMemory.getSerializerManager();
    MycatPropertyConf conf = myCatMemory.getConf();
    DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
    /**
     * 1.schema ,模拟一个field字段值
     */
    int fieldCount = 3;
    ColMeta colMeta = null;
    Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("age", colMeta);
    OrderCol[] orderCols = new OrderCol[1];
    OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_ASC);
    orderCols[0] = orderCol;
    /**
     * 2 .PrefixComputer
     */
    StructType schema = new StructType(colMetaMap, fieldCount);
    schema.setOrderCols(orderCols);
    UnsafeExternalRowSorter.PrefixComputer prefixComputer = new RowPrefixComputer(schema);
    /**
     * 3 .PrefixComparator 默认是ASC，可以选择DESC
     */
    final PrefixComparator prefixComparator = PrefixComparators.LONG;
    UnsafeExternalRowSorter sorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "1m"), true, /**
     *使用基数排序？true or false
     */
    true);
    UnsafeRow unsafeRow;
    BufferHolder bufferHolder;
    UnsafeRowWriter unsafeRowWriter;
    String line = "testUnsafeRow";
    // List<Float> floats = new ArrayList<Float>();
    List<Long> longs = new ArrayList<Long>();
    final Random rand = new Random(42);
    for (int i = 0; i < TEST_SIZE; i++) {
        unsafeRow = new UnsafeRow(3);
        bufferHolder = new BufferHolder(unsafeRow);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
        bufferHolder.reset();
        String key = getRandomString(rand.nextInt(300) + 100);
        // long v = rand.nextLong();
        // longs.add(v);
        unsafeRowWriter.write(0, key.getBytes());
        // unsafeRowWriter.write(0, BytesTools.toBytes(v));
        unsafeRowWriter.write(1, line.getBytes());
        unsafeRowWriter.write(2, ("35" + 1).getBytes());
        unsafeRow.setTotalSize(bufferHolder.totalSize());
        sorter.insertRow(unsafeRow);
    }
    Iterator<UnsafeRow> iter = sorter.sort();
    /*
         float [] com = new float[floats.size()];
        for (int i = 0; i <floats.size() ; i++) {
                com[i] = floats.get(i);
        }
        Arrays.sort(com);


        long[] com = new long[longs.size()];
        for (int i = 0; i < longs.size() ; i++) {
            com[i] = longs.get(i);
        }

        Arrays.sort(com);
        */
    UnsafeRow row = null;
    int indexprint = 0;
    while (iter.hasNext()) {
        row = iter.next();
        // LOGGER.error(indexprint + "    " +  row.getUTF8String(0));
        // Assert.assertEquals(com[indexprint],
        // BytesTools.toLong(row.getBinary(0)));
        // Double c = Double.parseDouble(String.valueOf(com[indexprint])) ;
        // Double c1 = Double.parseDouble(String.valueOf(BytesTools.toFloat(row.getBinary(0)))) ;
        // Assert.assertEquals(0,c.compareTo(c1));
        indexprint++;
    }
    Assert.assertEquals(TEST_SIZE, indexprint);
}
Also used : UnsafeExternalRowSorter(io.mycat.memory.unsafe.utils.sort.UnsafeExternalRowSorter) ColMeta(io.mycat.sqlengine.mpp.ColMeta) StructType(io.mycat.memory.unsafe.row.StructType) DataNodeDiskManager(io.mycat.memory.unsafe.storage.DataNodeDiskManager) MycatPropertyConf(io.mycat.memory.unsafe.utils.MycatPropertyConf) UnsafeRow(io.mycat.memory.unsafe.row.UnsafeRow) BufferHolder(io.mycat.memory.unsafe.row.BufferHolder) OrderCol(io.mycat.sqlengine.mpp.OrderCol) PrefixComparator(io.mycat.memory.unsafe.utils.sort.PrefixComparator) SerializerManager(io.mycat.memory.unsafe.storage.SerializerManager) DataNodeMemoryManager(io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager) UnsafeRowWriter(io.mycat.memory.unsafe.row.UnsafeRowWriter) DataNodeMemoryManager(io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager) MemoryManager(io.mycat.memory.unsafe.memory.mm.MemoryManager) RowPrefixComputer(io.mycat.memory.unsafe.utils.sort.RowPrefixComputer) MyCatMemory(io.mycat.memory.MyCatMemory) Test(org.junit.Test)

Example 4 with UnsafeExternalRowSorter

use of io.mycat.memory.unsafe.utils.sort.UnsafeExternalRowSorter in project Mycat_plus by coderczp.

the class DataNodeMergeManager method onRowMetaData.

public void onRowMetaData(Map<String, ColMeta> columToIndx, int fieldCount) throws IOException {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("field metadata keys:{}", columToIndx.keySet());
        LOGGER.debug("field metadata values:", columToIndx.values());
    }
    OrderCol[] orderCols = null;
    StructType schema = null;
    UnsafeExternalRowSorter.PrefixComputer prefixComputer = null;
    PrefixComparator prefixComparator = null;
    DataNodeMemoryManager dataNodeMemoryManager = null;
    UnsafeExternalRowSorter sorter = null;
    int[] groupColumnIndexs = null;
    this.fieldCount = fieldCount;
    if (rrs.getGroupByCols() != null) {
        groupColumnIndexs = toColumnIndex(rrs.getGroupByCols(), columToIndx);
        if (LOGGER.isDebugEnabled()) {
            for (int i = 0; i < rrs.getGroupByCols().length; i++) {
                LOGGER.debug("groupColumnIndexs:" + rrs.getGroupByCols()[i]);
            }
        }
    }
    if (rrs.getHavingCols() != null) {
        ColMeta colMeta = columToIndx.get(rrs.getHavingCols().getLeft().toUpperCase());
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("getHavingCols:" + rrs.getHavingCols().toString());
        }
        /**
         * mycat 中将 sql： select avg(xxx) from t
         * 重写 为 select sum(xxx) AS AVG[0~9]SUM,count(xxx) AS AVG[0~9]COUNT from t
         *  或者 select avg(xxx)  AS xxx from t
         *  select sum(xxx) AS xxxSUM,count(xxx) AS xxxCOUNT from t
         */
        if (colMeta == null) {
            for (String key : columToIndx.keySet()) {
                if (key.toUpperCase().endsWith("SUM")) {
                    colMeta = columToIndx.get(key);
                    break;
                }
            }
        }
        if (colMeta != null) {
            rrs.getHavingCols().setColMeta(colMeta);
        }
    }
    if (rrs.isHasAggrColumn()) {
        List<MergeCol> mergCols = new LinkedList<MergeCol>();
        Map<String, Integer> mergeColsMap = rrs.getMergeCols();
        if (mergeColsMap != null) {
            if (LOGGER.isDebugEnabled() && rrs.getMergeCols() != null) {
                LOGGER.debug("isHasAggrColumn:" + rrs.getMergeCols().toString());
            }
            for (Map.Entry<String, Integer> mergEntry : mergeColsMap.entrySet()) {
                String colName = mergEntry.getKey().toUpperCase();
                int type = mergEntry.getValue();
                if (MergeCol.MERGE_AVG == type) {
                    ColMeta sumColMeta = columToIndx.get(colName + "SUM");
                    ColMeta countColMeta = columToIndx.get(colName + "COUNT");
                    if (sumColMeta != null && countColMeta != null) {
                        ColMeta colMeta = new ColMeta(sumColMeta.colIndex, countColMeta.colIndex, sumColMeta.getColType());
                        mergCols.add(new MergeCol(colMeta, mergEntry.getValue()));
                    }
                } else {
                    ColMeta colMeta = columToIndx.get(colName);
                    mergCols.add(new MergeCol(colMeta, mergEntry.getValue()));
                }
            }
        }
        // add no alias merg column
        for (Map.Entry<String, ColMeta> fieldEntry : columToIndx.entrySet()) {
            String colName = fieldEntry.getKey();
            int result = MergeCol.tryParseAggCol(colName);
            if (result != MergeCol.MERGE_UNSUPPORT && result != MergeCol.MERGE_NOMERGE) {
                mergCols.add(new MergeCol(fieldEntry.getValue(), result));
            }
        }
        /**
         * Group操作
         */
        MergeCol[] mergColsArrays = mergCols.toArray(new MergeCol[mergCols.size()]);
        unsafeRowGrouper = new UnsafeRowGrouper(columToIndx, rrs.getGroupByCols(), mergColsArrays, rrs.getHavingCols());
        if (mergColsArrays != null && mergColsArrays.length > 0) {
            mergeColsIndex = new int[mergColsArrays.length];
            for (int i = 0; i < mergColsArrays.length; i++) {
                mergeColsIndex[i] = mergColsArrays[i].colMeta.colIndex;
            }
            Arrays.sort(mergeColsIndex);
        }
    }
    if (rrs.getOrderByCols() != null) {
        LinkedHashMap<String, Integer> orders = rrs.getOrderByCols();
        orderCols = new OrderCol[orders.size()];
        int i = 0;
        for (Map.Entry<String, Integer> entry : orders.entrySet()) {
            String key = StringUtil.removeBackquote(entry.getKey().toUpperCase());
            ColMeta colMeta = columToIndx.get(key);
            if (colMeta == null) {
                throw new IllegalArgumentException("all columns in order by clause should be in the selected column list!" + entry.getKey());
            }
            orderCols[i++] = new OrderCol(colMeta, entry.getValue());
        }
        /**
         * 构造全局排序器
         */
        schema = new StructType(columToIndx, fieldCount);
        schema.setOrderCols(orderCols);
        prefixComputer = new RowPrefixComputer(schema);
        // if(orderCols.length>0
        // && orderCols[0].getOrderType()
        // == OrderCol.COL_ORDER_TYPE_ASC){
        // prefixComparator = PrefixComparators.LONG;
        // }else {
        // prefixComparator = PrefixComparators.LONG_DESC;
        // }
        prefixComparator = getPrefixComparator(orderCols);
        dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
        /**
         * 默认排序，只是将数据连续存储到内存中即可。
         */
        globalSorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "32k"), false, /**
         *是否使用基数排序
         */
        true);
    }
    if (conf.getBoolean("mycat.stream.output.result", false) && globalSorter == null && unsafeRowGrouper == null) {
        setStreamOutputResult(true);
    } else {
        /**
         * 1.schema
         */
        schema = new StructType(columToIndx, fieldCount);
        schema.setOrderCols(orderCols);
        /**
         * 2 .PrefixComputer
         */
        prefixComputer = new RowPrefixComputer(schema);
        /**
         * 3 .PrefixComparator 默认是ASC，可以选择DESC
         */
        prefixComparator = PrefixComparators.LONG;
        dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
        globalMergeResult = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "32k"), false, /**
         *是否使用基数排序
         */
        false);
    }
}
Also used : PrefixComparator(io.mycat.memory.unsafe.utils.sort.PrefixComparator) UnsafeExternalRowSorter(io.mycat.memory.unsafe.utils.sort.UnsafeExternalRowSorter) StructType(io.mycat.memory.unsafe.row.StructType) DataNodeMemoryManager(io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager) LinkedList(java.util.LinkedList) RowPrefixComputer(io.mycat.memory.unsafe.utils.sort.RowPrefixComputer) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap)

Aggregations

DataNodeMemoryManager (io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager)4 StructType (io.mycat.memory.unsafe.row.StructType)4 PrefixComparator (io.mycat.memory.unsafe.utils.sort.PrefixComparator)4 RowPrefixComputer (io.mycat.memory.unsafe.utils.sort.RowPrefixComputer)4 UnsafeExternalRowSorter (io.mycat.memory.unsafe.utils.sort.UnsafeExternalRowSorter)4 MyCatMemory (io.mycat.memory.MyCatMemory)2 MemoryManager (io.mycat.memory.unsafe.memory.mm.MemoryManager)2 BufferHolder (io.mycat.memory.unsafe.row.BufferHolder)2 UnsafeRow (io.mycat.memory.unsafe.row.UnsafeRow)2 UnsafeRowWriter (io.mycat.memory.unsafe.row.UnsafeRowWriter)2 DataNodeDiskManager (io.mycat.memory.unsafe.storage.DataNodeDiskManager)2 SerializerManager (io.mycat.memory.unsafe.storage.SerializerManager)2 MycatPropertyConf (io.mycat.memory.unsafe.utils.MycatPropertyConf)2 ColMeta (io.mycat.sqlengine.mpp.ColMeta)2 OrderCol (io.mycat.sqlengine.mpp.OrderCol)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 Test (org.junit.Test)2 LinkedHashMap (java.util.LinkedHashMap)1 LinkedList (java.util.LinkedList)1 Map (java.util.Map)1
About Me
UseOf

Java Tutorials :

Simple Java RabbitMQ Example with Spring
Simple Springboot JPA Eclipeslink Example
Simple SpringBoot JPA Hibernate Example
Jackson JSON @JsonIgnore and @JsonIgnoreProperties Examples
Java Infinite recursion (StackOverflowError) Jackson solutions
Simple Java Jackson Serialize Objects to JSON and convert them back To Object
ElasticSearch 5 - Upload files using Java API in binary field Example
Java JAXB marshall unmarshall Examples from String and Stream
Java 8 forEach List and Map examples
ElasticSearch 5 Java API SearchRequestBuilder examples
Java ElasticSearch 5 examples with node index and mapping - running local
Convert String to int or Integer Java 8
Java 9 in action - JShell - Ubuntu
Java - removing invalid characters from a file name
Hello world!? or Hello Tutorial