Examples with ColMeta io.mycat.sqlengine.mpp.ColMeta used on opensource projects

Example 6 with ColMeta

use of io.mycat.sqlengine.mpp.ColMeta in project Mycat-Server by MyCATApache.

the class TestSorter method runSorter.

public void runSorter(MyCatMemory myCatMemory, MemoryManager memoryManager, MycatPropertyConf conf) throws NoSuchFieldException, IllegalAccessException, IOException {
    DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
    /**
     * 1.schema ,模拟一个field字段值
     */
    int fieldCount = 3;
    ColMeta colMeta = null;
    Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("age", colMeta);
    OrderCol[] orderCols = new OrderCol[1];
    OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_ASC);
    orderCols[0] = orderCol;
    /**
     * 2 .PrefixComputer
     */
    StructType schema = new StructType(colMetaMap, fieldCount);
    schema.setOrderCols(orderCols);
    UnsafeExternalRowSorter.PrefixComputer prefixComputer = new RowPrefixComputer(schema);
    /**
     * 3 .PrefixComparator 默认是ASC，可以选择DESC
     */
    final PrefixComparator prefixComparator = PrefixComparators.LONG;
    UnsafeExternalRowSorter sorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "1m"), true, /**
     *使用基数排序？true or false
     */
    true);
    UnsafeRow unsafeRow;
    BufferHolder bufferHolder;
    UnsafeRowWriter unsafeRowWriter;
    String line = "testUnsafeRow";
    final Random rand = new Random(42);
    for (int i = 0; i < TEST_SIZE; i++) {
        unsafeRow = new UnsafeRow(3);
        bufferHolder = new BufferHolder(unsafeRow);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
        bufferHolder.reset();
        String key = getRandomString(rand.nextInt(300) + 100);
        unsafeRowWriter.write(0, key.getBytes());
        unsafeRowWriter.write(1, line.getBytes());
        unsafeRowWriter.write(2, ("35" + 1).getBytes());
        unsafeRow.setTotalSize(bufferHolder.totalSize());
        sorter.insertRow(unsafeRow);
    }
    Iterator<UnsafeRow> iter = sorter.sort();
    UnsafeRow row = null;
    int indexprint = 0;
    while (iter.hasNext()) {
        row = iter.next();
        indexprint++;
    }
    sorter.cleanupResources();
    countDownLatch.countDown();
    System.out.println("Thread ID :" + Thread.currentThread().getId() + "Index : " + indexprint);
}

Example 7 with ColMeta

use of io.mycat.sqlengine.mpp.ColMeta in project Mycat_plus by coderczp.

the class TestSorter method runSorter.

public void runSorter(MyCatMemory myCatMemory, MemoryManager memoryManager, MycatPropertyConf conf) throws NoSuchFieldException, IllegalAccessException, IOException {
    DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
    /**
     * 1.schema ,模拟一个field字段值
     */
    int fieldCount = 3;
    ColMeta colMeta = null;
    Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("age", colMeta);
    OrderCol[] orderCols = new OrderCol[1];
    OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_ASC);
    orderCols[0] = orderCol;
    /**
     * 2 .PrefixComputer
     */
    StructType schema = new StructType(colMetaMap, fieldCount);
    schema.setOrderCols(orderCols);
    UnsafeExternalRowSorter.PrefixComputer prefixComputer = new RowPrefixComputer(schema);
    /**
     * 3 .PrefixComparator 默认是ASC，可以选择DESC
     */
    final PrefixComparator prefixComparator = PrefixComparators.LONG;
    UnsafeExternalRowSorter sorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "1m"), true, /**
     *使用基数排序？true or false
     */
    true);
    UnsafeRow unsafeRow;
    BufferHolder bufferHolder;
    UnsafeRowWriter unsafeRowWriter;
    String line = "testUnsafeRow";
    final Random rand = new Random(42);
    for (int i = 0; i < TEST_SIZE; i++) {
        unsafeRow = new UnsafeRow(3);
        bufferHolder = new BufferHolder(unsafeRow);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
        bufferHolder.reset();
        String key = getRandomString(rand.nextInt(300) + 100);
        unsafeRowWriter.write(0, key.getBytes());
        unsafeRowWriter.write(1, line.getBytes());
        unsafeRowWriter.write(2, ("35" + 1).getBytes());
        unsafeRow.setTotalSize(bufferHolder.totalSize());
        sorter.insertRow(unsafeRow);
    }
    Iterator<UnsafeRow> iter = sorter.sort();
    UnsafeRow row = null;
    int indexprint = 0;
    while (iter.hasNext()) {
        row = iter.next();
        indexprint++;
    }
    sorter.cleanupResources();
    countDownLatch.countDown();
    System.out.println("Thread ID :" + Thread.currentThread().getId() + "Index : " + indexprint);
}

Example 8 with ColMeta

use of io.mycat.sqlengine.mpp.ColMeta in project Mycat_plus by coderczp.

the class MultiNodeQueryHandler method fieldEofResponse.

@Override
public void fieldEofResponse(byte[] header, List<byte[]> fields, byte[] eof, BackendConnection conn) {
    // huangyiming add
    this.header = header;
    this.fields = fields;
    MiddlerResultHandler middlerResultHandler = session.getMiddlerResultHandler();
    /*if(null !=middlerResultHandler ){
			return;
		}*/
    this.netOutBytes += header.length;
    this.netOutBytes += eof.length;
    for (int i = 0, len = fields.size(); i < len; ++i) {
        byte[] field = fields.get(i);
        this.netOutBytes += field.length;
    }
    ServerConnection source = null;
    if (fieldsReturned) {
        return;
    }
    lock.lock();
    try {
        if (fieldsReturned) {
            return;
        }
        fieldsReturned = true;
        boolean needMerg = (dataMergeSvr != null) && dataMergeSvr.getRrs().needMerge();
        Set<String> shouldRemoveAvgField = new HashSet<>();
        Set<String> shouldRenameAvgField = new HashSet<>();
        if (needMerg) {
            Map<String, Integer> mergeColsMap = dataMergeSvr.getRrs().getMergeCols();
            if (mergeColsMap != null) {
                for (Map.Entry<String, Integer> entry : mergeColsMap.entrySet()) {
                    String key = entry.getKey();
                    int mergeType = entry.getValue();
                    if (MergeCol.MERGE_AVG == mergeType && mergeColsMap.containsKey(key + "SUM")) {
                        shouldRemoveAvgField.add((key + "COUNT").toUpperCase());
                        shouldRenameAvgField.add((key + "SUM").toUpperCase());
                    }
                }
            }
        }
        source = session.getSource();
        ByteBuffer buffer = source.allocate();
        fieldCount = fields.size();
        if (shouldRemoveAvgField.size() > 0) {
            ResultSetHeaderPacket packet = new ResultSetHeaderPacket();
            packet.packetId = ++packetId;
            packet.fieldCount = fieldCount - shouldRemoveAvgField.size();
            buffer = packet.write(buffer, source, true);
        } else {
            header[3] = ++packetId;
            buffer = source.writeToBuffer(header, buffer);
        }
        String primaryKey = null;
        if (rrs.hasPrimaryKeyToCache()) {
            String[] items = rrs.getPrimaryKeyItems();
            priamaryKeyTable = items[0];
            primaryKey = items[1];
        }
        Map<String, ColMeta> columToIndx = new HashMap<String, ColMeta>(fieldCount);
        for (int i = 0, len = fieldCount; i < len; ++i) {
            boolean shouldSkip = false;
            byte[] field = fields.get(i);
            if (needMerg) {
                FieldPacket fieldPkg = new FieldPacket();
                fieldPkg.read(field);
                fieldPackets.add(fieldPkg);
                String fieldName = new String(fieldPkg.name).toUpperCase();
                if (columToIndx != null && !columToIndx.containsKey(fieldName)) {
                    if (shouldRemoveAvgField.contains(fieldName)) {
                        shouldSkip = true;
                        fieldPackets.remove(fieldPackets.size() - 1);
                    }
                    if (shouldRenameAvgField.contains(fieldName)) {
                        String newFieldName = fieldName.substring(0, fieldName.length() - 3);
                        fieldPkg.name = newFieldName.getBytes();
                        fieldPkg.packetId = ++packetId;
                        shouldSkip = true;
                        // 处理AVG字段位数和精度, AVG位数 = SUM位数 - 14
                        fieldPkg.length = fieldPkg.length - 14;
                        // AVG精度 = SUM精度 + 4
                        fieldPkg.decimals = (byte) (fieldPkg.decimals + 4);
                        buffer = fieldPkg.write(buffer, source, false);
                        // 还原精度
                        fieldPkg.decimals = (byte) (fieldPkg.decimals - 4);
                    }
                    ColMeta colMeta = new ColMeta(i, fieldPkg.type);
                    colMeta.decimals = fieldPkg.decimals;
                    columToIndx.put(fieldName, colMeta);
                }
            } else {
                FieldPacket fieldPkg = new FieldPacket();
                fieldPkg.read(field);
                fieldPackets.add(fieldPkg);
                fieldCount = fields.size();
                if (primaryKey != null && primaryKeyIndex == -1) {
                    // find primary key index
                    String fieldName = new String(fieldPkg.name);
                    if (primaryKey.equalsIgnoreCase(fieldName)) {
                        primaryKeyIndex = i;
                    }
                }
            }
            if (!shouldSkip) {
                field[3] = ++packetId;
                buffer = source.writeToBuffer(field, buffer);
            }
        }
        eof[3] = ++packetId;
        buffer = source.writeToBuffer(eof, buffer);
        if (null == middlerResultHandler) {
            // session.getSource().write(row);
            source.write(buffer);
        }
        if (dataMergeSvr != null) {
            dataMergeSvr.onRowMetaData(columToIndx, fieldCount);
        }
    } catch (Exception e) {
        handleDataProcessException(e);
    } finally {
        lock.unlock();
    }
}

Example 9 with ColMeta

use of io.mycat.sqlengine.mpp.ColMeta in project Mycat-Server by MyCATApache.

the class MultiNodeQueryHandler method fieldEofResponse.

@Override
public void fieldEofResponse(byte[] header, List<byte[]> fields, byte[] eof, BackendConnection conn) {
    // huangyiming add
    this.header = header;
    this.fields = fields;
    MiddlerResultHandler middlerResultHandler = session.getMiddlerResultHandler();
    /*if(null !=middlerResultHandler ){
			return;
		}*/
    this.netOutBytes += header.length;
    this.netOutBytes += eof.length;
    for (int i = 0, len = fields.size(); i < len; ++i) {
        byte[] field = fields.get(i);
        this.netOutBytes += field.length;
    }
    ServerConnection source = null;
    if (fieldsReturned) {
        return;
    }
    lock.lock();
    try {
        if (fieldsReturned) {
            return;
        }
        fieldsReturned = true;
        boolean needMerg = (dataMergeSvr != null) && dataMergeSvr.getRrs().needMerge();
        Set<String> shouldRemoveAvgField = new HashSet<>();
        Set<String> shouldRenameAvgField = new HashSet<>();
        if (needMerg) {
            Map<String, Integer> mergeColsMap = dataMergeSvr.getRrs().getMergeCols();
            if (mergeColsMap != null) {
                for (Map.Entry<String, Integer> entry : mergeColsMap.entrySet()) {
                    String key = entry.getKey();
                    int mergeType = entry.getValue();
                    if (MergeCol.MERGE_AVG == mergeType && mergeColsMap.containsKey(key + "SUM")) {
                        shouldRemoveAvgField.add((key + "COUNT").toUpperCase());
                        shouldRenameAvgField.add((key + "SUM").toUpperCase());
                    }
                }
            }
        }
        source = session.getSource();
        ByteBuffer buffer = source.allocate();
        fieldCount = fields.size();
        if (shouldRemoveAvgField.size() > 0) {
            ResultSetHeaderPacket packet = new ResultSetHeaderPacket();
            packet.packetId = ++packetId;
            packet.fieldCount = fieldCount - shouldRemoveAvgField.size();
            buffer = packet.write(buffer, source, true);
        } else {
            header[3] = ++packetId;
            buffer = source.writeToBuffer(header, buffer);
        }
        String primaryKey = null;
        if (rrs.hasPrimaryKeyToCache()) {
            String[] items = rrs.getPrimaryKeyItems();
            priamaryKeyTable = items[0];
            primaryKey = items[1];
        }
        Map<String, ColMeta> columToIndx = new HashMap<String, ColMeta>(fieldCount);
        for (int i = 0, len = fieldCount; i < len; ++i) {
            boolean shouldSkip = false;
            byte[] field = fields.get(i);
            if (needMerg) {
                FieldPacket fieldPkg = new FieldPacket();
                fieldPkg.read(field);
                fieldPackets.add(fieldPkg);
                String fieldName = new String(fieldPkg.name).toUpperCase();
                if (columToIndx != null && !columToIndx.containsKey(fieldName)) {
                    if (shouldRemoveAvgField.contains(fieldName)) {
                        shouldSkip = true;
                        fieldPackets.remove(fieldPackets.size() - 1);
                    }
                    if (shouldRenameAvgField.contains(fieldName)) {
                        String newFieldName = fieldName.substring(0, fieldName.length() - 3);
                        fieldPkg.name = newFieldName.getBytes();
                        fieldPkg.packetId = ++packetId;
                        shouldSkip = true;
                        // 处理AVG字段位数和精度, AVG位数 = SUM位数 - 14
                        fieldPkg.length = fieldPkg.length - 14;
                        // AVG精度 = SUM精度 + 4
                        fieldPkg.decimals = (byte) (fieldPkg.decimals + 4);
                        buffer = fieldPkg.write(buffer, source, false);
                        // 还原精度
                        fieldPkg.decimals = (byte) (fieldPkg.decimals - 4);
                    }
                    ColMeta colMeta = new ColMeta(i, fieldPkg.type);
                    colMeta.decimals = fieldPkg.decimals;
                    columToIndx.put(fieldName, colMeta);
                }
            } else {
                FieldPacket fieldPkg = new FieldPacket();
                fieldPkg.read(field);
                fieldPackets.add(fieldPkg);
                fieldCount = fields.size();
                if (primaryKey != null && primaryKeyIndex == -1) {
                    // find primary key index
                    String fieldName = new String(fieldPkg.name);
                    if (primaryKey.equalsIgnoreCase(fieldName)) {
                        primaryKeyIndex = i;
                    }
                }
            }
            if (!shouldSkip) {
                field[3] = ++packetId;
                buffer = source.writeToBuffer(field, buffer);
            }
        }
        eof[3] = ++packetId;
        buffer = source.writeToBuffer(eof, buffer);
        if (null == middlerResultHandler) {
            // session.getSource().write(row);
            source.write(buffer);
        }
        if (dataMergeSvr != null) {
            dataMergeSvr.onRowMetaData(columToIndx, fieldCount);
        }
    } catch (Exception e) {
        handleDataProcessException(e);
    } finally {
        lock.unlock();
    }
}

Example 10 with ColMeta

use of io.mycat.sqlengine.mpp.ColMeta in project Mycat-Server by MyCATApache.

the class UnsafeFixedWidthAggregationMapSuite method testWithMemoryLeakDetection.

@Test
public void testWithMemoryLeakDetection() throws IOException, NoSuchFieldException, IllegalAccessException {
    MyCatMemory myCatMemory = new MyCatMemory();
    MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager();
    DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
    int fieldCount = 3;
    ColMeta colMeta = null;
    Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("age", colMeta);
    OrderCol[] orderCols = new OrderCol[1];
    OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_DESC);
    orderCols[0] = orderCol;
    groupKeySchema = new StructType(colMetaMap, fieldCount);
    groupKeySchema.setOrderCols(orderCols);
    fieldCount = 3;
    colMeta = null;
    colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_LONGLONG);
    colMetaMap.put("age", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_LONGLONG);
    colMetaMap.put("age1", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    orderCols = new OrderCol[1];
    orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_DESC);
    orderCols[0] = orderCol;
    aggBufferSchema = new StructType(colMetaMap, fieldCount);
    aggBufferSchema.setOrderCols(orderCols);
    /**
     * value
     */
    BufferHolder bufferHolder;
    emptyAggregationBuffer = new UnsafeRow(3);
    bufferHolder = new BufferHolder(emptyAggregationBuffer, 0);
    UnsafeRowWriter unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
    bufferHolder.reset();
    String value = "ok,hello";
    emptyAggregationBuffer.setLong(0, 0);
    emptyAggregationBuffer.setLong(1, 0);
    unsafeRowWriter.write(2, value.getBytes());
    emptyAggregationBuffer.setTotalSize(bufferHolder.totalSize());
    UnsafeFixedWidthAggregationMap map = new UnsafeFixedWidthAggregationMap(emptyAggregationBuffer, aggBufferSchema, groupKeySchema, dataNodeMemoryManager, 2 * 1024, PAGE_SIZE_BYTES, false);
    int i;
    List<UnsafeRow> rows = new ArrayList<UnsafeRow>();
    for (i = 0; i < 1000; i++) {
        String line = "testUnsafeRow" + i;
        /**
         * key
         */
        UnsafeRow groupKey = new UnsafeRow(3);
        bufferHolder = new BufferHolder(groupKey, 0);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
        bufferHolder.reset();
        final byte[] key = getRandomByteArray(rand.nextInt(8));
        String age = "5" + i;
        unsafeRowWriter.write(0, key);
        unsafeRowWriter.write(1, line.getBytes());
        unsafeRowWriter.write(2, age.getBytes());
        groupKey.setTotalSize(bufferHolder.totalSize());
        map.getAggregationBuffer(groupKey);
        rows.add(groupKey);
    }
    Assert.assertEquals(i, rows.size());
    UnsafeRow row = rows.get(12);
    UnsafeRow rs = map.getAggregationBuffer(row);
    rs.setLong(0, 12);
    rs = map.getAggregationBuffer(row);
    Assert.assertEquals(12, rs.getLong(0));
    map.free();
}