Examples with UnsafeRow io.mycat.memory.unsafe.row.UnsafeRow used on opensource projects

Example 11 with UnsafeRow

use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.

the class UnsafeFixedWidthAggregationMapSuite method testAggregateMap.

@Test
public void testAggregateMap() throws NoSuchFieldException, IllegalAccessException, IOException {
    /**
     * 创造上文环境
     */
    MyCatMemory myCatMemory = new MyCatMemory();
    MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager();
    DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
    /**
       * 构造数据字段group key
       */
    int fieldCount = 2;
    ColMeta colMeta = null;
    Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    OrderCol[] orderCols = new OrderCol[1];
    OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_DESC);
    orderCols[0] = orderCol;
    groupKeySchema = new StructType(colMetaMap, fieldCount);
    groupKeySchema.setOrderCols(orderCols);
    /**
   * 构造数据字段value key
   */
    fieldCount = 4;
    colMeta = null;
    colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_INT);
    colMetaMap.put("age", colMeta);
    colMeta = new ColMeta(3, ColMeta.COL_TYPE_LONGLONG);
    colMetaMap.put("score", colMeta);
    orderCols = new OrderCol[1];
    orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_DESC);
    orderCols[0] = orderCol;
    aggBufferSchema = new StructType(colMetaMap, fieldCount);
    aggBufferSchema.setOrderCols(orderCols);
    /**
     *emtpy Row value
     */
    BufferHolder bufferHolder;
    emptyAggregationBuffer = new UnsafeRow(4);
    bufferHolder = new BufferHolder(emptyAggregationBuffer, 0);
    UnsafeRowWriter unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 4);
    bufferHolder.reset();
    String value = "o";
    unsafeRowWriter.write(0, value.getBytes());
    unsafeRowWriter.write(1, value.getBytes());
    emptyAggregationBuffer.setInt(2, 0);
    emptyAggregationBuffer.setLong(3, 0);
    emptyAggregationBuffer.setTotalSize(bufferHolder.totalSize());
    UnsafeFixedWidthAggregationMap map = new UnsafeFixedWidthAggregationMap(emptyAggregationBuffer, aggBufferSchema, groupKeySchema, dataNodeMemoryManager, 2 * 1024, PAGE_SIZE_BYTES, true);
    /**
       * 造数据
       */
    int i;
    List<UnsafeRow> rows = new ArrayList<UnsafeRow>();
    for (i = 0; i < 100000; i++) {
        /**
       * key
       */
        UnsafeRow groupKey = new UnsafeRow(2);
        bufferHolder = new BufferHolder(groupKey, 0);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 2);
        bufferHolder.reset();
        unsafeRowWriter.write(0, BytesTools.toBytes(rand.nextInt(10000000)));
        unsafeRowWriter.write(1, BytesTools.toBytes(rand.nextInt(10000000)));
        groupKey.setTotalSize(bufferHolder.totalSize());
        UnsafeRow valueKey = new UnsafeRow(4);
        bufferHolder = new BufferHolder(valueKey, 0);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 4);
        bufferHolder.reset();
        unsafeRowWriter.write(0, BytesTools.toBytes(rand.nextInt(10)));
        unsafeRowWriter.write(1, BytesTools.toBytes(rand.nextInt(10)));
        valueKey.setInt(2, i);
        valueKey.setLong(3, 1);
        valueKey.setTotalSize(bufferHolder.totalSize());
        if (map.find(groupKey)) {
            UnsafeRow rs = map.getAggregationBuffer(groupKey);
            rs.setLong(3, i + valueKey.getLong(3));
            rs.setInt(2, 100 + valueKey.getInt(2));
        } else {
            map.put(groupKey, valueKey);
        }
        rows.add(valueKey);
    }
    KVIterator<UnsafeRow, UnsafeRow> iter = map.iterator();
    int j = 0;
    while (iter.next()) {
        Assert.assertEquals(j, iter.getValue().getInt(2));
        j++;
        iter.getValue().setInt(2, 5000000);
        iter.getValue().setLong(3, 600000);
    }
    Assert.assertEquals(rows.size(), j);
    int k = 0;
    KVIterator<UnsafeRow, UnsafeRow> iter1 = map.iterator();
    while (iter1.next()) {
        k++;
        // LOGGER.error("(" + BytesTools.toInt(iter1.getKey().getBinary(0)) + "," +
        //       iter1.getValue().getInt(2) +"," +iter1.getValue().getLong(3)+")");
        Assert.assertEquals(5000000, iter1.getValue().getInt(2));
        Assert.assertEquals(600000, iter1.getValue().getLong(3));
    }
    Assert.assertEquals(j, k);
    map.free();
}

Example 12 with UnsafeRow

use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.

the class MultiNodeQueryHandler method outputMergeResult.

/**
	 * 将汇聚结果集数据真正的发送给Mycat客户端
	 * @param source
	 * @param eof
	 * @param
	 */
public void outputMergeResult(final ServerConnection source, final byte[] eof, Iterator<UnsafeRow> iter) {
    try {
        lock.lock();
        ByteBuffer buffer = session.getSource().allocate();
        final RouteResultset rrs = this.dataMergeSvr.getRrs();
        /**
			 * 处理limit语句的start 和 end位置，将正确的结果发送给
			 * Mycat 客户端
			 */
        int start = rrs.getLimitStart();
        int end = start + rrs.getLimitSize();
        int index = 0;
        if (start < 0)
            start = 0;
        if (rrs.getLimitSize() < 0)
            end = Integer.MAX_VALUE;
        if (prepared) {
            while (iter.hasNext()) {
                UnsafeRow row = iter.next();
                if (index >= start) {
                    row.packetId = ++packetId;
                    BinaryRowDataPacket binRowPacket = new BinaryRowDataPacket();
                    binRowPacket.read(fieldPackets, row);
                    buffer = binRowPacket.write(buffer, source, true);
                }
                index++;
                if (index == end) {
                    break;
                }
            }
        } else {
            while (iter.hasNext()) {
                UnsafeRow row = iter.next();
                if (index >= start) {
                    row.packetId = ++packetId;
                    buffer = row.write(buffer, source, true);
                }
                index++;
                if (index == end) {
                    break;
                }
            }
        }
        eof[3] = ++packetId;
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("last packet id:" + packetId);
        }
        /**
			 * 真正的开始把Writer Buffer的数据写入到channel 中
			 */
        source.write(source.writeToBuffer(eof, buffer));
    } catch (Exception e) {
        handleDataProcessException(e);
    } finally {
        lock.unlock();
        dataMergeSvr.clear();
    }
}

Example 13 with UnsafeRow

use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.

the class DataNodeMergeManager method run.

@Override
public void run() {
    if (!running.compareAndSet(false, true)) {
        return;
    }
    boolean nulpack = false;
    try {
        for (; ; ) {
            final PackWraper pack = packs.poll();
            if (pack == null) {
                nulpack = true;
                break;
            }
            if (pack == END_FLAG_PACK) {
                /**
                     * 最后一个节点datenode发送了row eof packet说明了整个
                     * 分片数据全部接收完成，进而将结果集全部发给你Mycat 客户端
                     */
                final int warningCount = 0;
                final EOFPacket eofp = new EOFPacket();
                final ByteBuffer eof = ByteBuffer.allocate(9);
                BufferUtil.writeUB3(eof, eofp.calcPacketSize());
                eof.put(eofp.packetId);
                eof.put(eofp.fieldCount);
                BufferUtil.writeUB2(eof, warningCount);
                BufferUtil.writeUB2(eof, eofp.status);
                final ServerConnection source = multiQueryHandler.getSession().getSource();
                final byte[] array = eof.array();
                Iterator<UnsafeRow> iters = null;
                if (unsafeRowGrouper != null) {
                    /**
                         * group by里面需要排序情况
                         */
                    if (globalSorter != null) {
                        iters = unsafeRowGrouper.getResult(globalSorter);
                    } else {
                        iters = unsafeRowGrouper.getResult(globalMergeResult);
                    }
                } else if (globalSorter != null) {
                    iters = globalSorter.sort();
                } else if (!isStreamOutputResult) {
                    iters = globalMergeResult.sort();
                }
                if (iters != null)
                    multiQueryHandler.outputMergeResult(source, array, iters);
                if (unsafeRowGrouper != null) {
                    unsafeRowGrouper.free();
                    unsafeRowGrouper = null;
                }
                if (globalSorter != null) {
                    globalSorter.cleanupResources();
                    globalSorter = null;
                }
                if (globalMergeResult != null) {
                    globalMergeResult.cleanupResources();
                    globalMergeResult = null;
                }
                break;
            }
            unsafeRow = new UnsafeRow(fieldCount);
            bufferHolder = new BufferHolder(unsafeRow, 0);
            unsafeRowWriter = new UnsafeRowWriter(bufferHolder, fieldCount);
            bufferHolder.reset();
            /**
                 *构造一行row，将对应的col填充.
                 */
            MySQLMessage mm = new MySQLMessage(pack.rowData);
            mm.readUB3();
            mm.read();
            for (int i = 0; i < fieldCount; i++) {
                byte[] colValue = mm.readBytesWithLength();
                if (colValue != null)
                    unsafeRowWriter.write(i, colValue);
                else
                    unsafeRow.setNullAt(i);
            }
            unsafeRow.setTotalSize(bufferHolder.totalSize());
            if (unsafeRowGrouper != null) {
                unsafeRowGrouper.addRow(unsafeRow);
            } else if (globalSorter != null) {
                globalSorter.insertRow(unsafeRow);
            } else {
                globalMergeResult.insertRow(unsafeRow);
            }
            unsafeRow = null;
            bufferHolder = null;
            unsafeRowWriter = null;
        }
    } catch (final Exception e) {
        multiQueryHandler.handleDataProcessException(e);
    } finally {
        running.set(false);
        if (nulpack && !packs.isEmpty()) {
            this.run();
        }
    }
}

Example 14 with UnsafeRow

use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.

the class UnsafeRowGrouper method insertValue.

private void insertValue(@Nonnull UnsafeExternalRowSorter sorter) {
    KVIterator<UnsafeRow, UnsafeRow> it = aggregationMap.iterator();
    try {
        while (it.next()) {
            UnsafeRow row = getAllBinaryRow(it.getValue());
            sorter.insertRow(row);
        }
    } catch (IOException e) {
        logger.error(e.getMessage());
    }
}

Example 15 with UnsafeRow

use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.

the class UnsafeRowGrouper method getGroupKey.

/**
	 * 构造groupKey
	 */
private UnsafeRow getGroupKey(UnsafeRow row) throws UnsupportedEncodingException {
    UnsafeRow key = null;
    if (this.sortColumnsByIndex == null) {
        /**
             * 针对没有group by关键字
             * select count(*) from table;
             */
        key = new UnsafeRow(this.groupKeyfieldCount + 1);
        bufferHolder = new BufferHolder(key, 0);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, this.groupKeyfieldCount + 1);
        bufferHolder.reset();
        unsafeRowWriter.write(0, "same".getBytes());
        key.setTotalSize(bufferHolder.totalSize());
        return key;
    }
    key = new UnsafeRow(this.groupKeyfieldCount);
    bufferHolder = new BufferHolder(key, 0);
    unsafeRowWriter = new UnsafeRowWriter(bufferHolder, this.groupKeyfieldCount);
    bufferHolder.reset();
    ColMeta curColMeta = null;
    for (int i = 0; i < this.groupKeyfieldCount; i++) {
        curColMeta = this.columToIndx.get(sortColumnsByIndex[i].toUpperCase());
        if (!row.isNullAt(curColMeta.colIndex)) {
            switch(curColMeta.colType) {
                case ColMeta.COL_TYPE_BIT:
                    key.setByte(i, row.getByte(curColMeta.colIndex));
                case ColMeta.COL_TYPE_INT:
                case ColMeta.COL_TYPE_LONG:
                case ColMeta.COL_TYPE_INT24:
                    key.setInt(i, BytesTools.getInt(row.getBinary(curColMeta.colIndex)));
                    break;
                case ColMeta.COL_TYPE_SHORT:
                    key.setShort(i, BytesTools.getShort(row.getBinary(curColMeta.colIndex)));
                    break;
                case ColMeta.COL_TYPE_FLOAT:
                    key.setFloat(i, BytesTools.getFloat(row.getBinary(curColMeta.colIndex)));
                    break;
                case ColMeta.COL_TYPE_DOUBLE:
                    key.setDouble(i, BytesTools.getDouble(row.getBinary(curColMeta.colIndex)));
                    break;
                case ColMeta.COL_TYPE_NEWDECIMAL:
                    //						key.setDouble(i,
                    //								BytesTools.getDouble(row.getBinary(curColMeta.colIndex)));
                    unsafeRowWriter.write(i, new BigDecimal(new String(row.getBinary(curColMeta.colIndex))));
                    break;
                case ColMeta.COL_TYPE_LONGLONG:
                    key.setLong(i, BytesTools.getLong(row.getBinary(curColMeta.colIndex)));
                    break;
                default:
                    unsafeRowWriter.write(i, row.getBinary(curColMeta.colIndex));
                    break;
            }
        } else {
            key.setNullAt(i);
        }
    }
    key.setTotalSize(bufferHolder.totalSize());
    return key;
}