Examples with UnsafeRow io.mycat.memory.unsafe.row.UnsafeRow used on opensource projects

Example 1 with UnsafeRow

use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.

the class UnsafeFixedWidthAggregationMap method iterator.

/**
   * Returns an iterator over the keys and values in this map. This uses destructive iterator of
   * BytesToBytesMap. So it is illegal to call any other method on this map after `iterator()` has
   * been called.
   *
   * For efficiency, each call returns the same object.
   */
public KVIterator<UnsafeRow, UnsafeRow> iterator() {
    return new KVIterator<UnsafeRow, UnsafeRow>() {

        private final BytesToBytesMap.MapIterator mapLocationIterator = map.iterator();

        private final UnsafeRow key = new UnsafeRow(groupingKeySchema.length());

        private final UnsafeRow value = new UnsafeRow(aggregationBufferSchema.length());

        @Override
        public boolean next() {
            if (mapLocationIterator.hasNext()) {
                final BytesToBytesMap.Location loc = mapLocationIterator.next();
                if (loc == null)
                    return false;
                key.pointTo(loc.getKeyBase(), loc.getKeyOffset(), loc.getKeyLength());
                value.pointTo(loc.getValueBase(), loc.getValueOffset(), loc.getValueLength());
                return true;
            } else {
                return false;
            }
        }

        @Override
        public UnsafeRow getKey() {
            return key;
        }

        @Override
        public UnsafeRow getValue() {
            return value;
        }

        @Override
        public void close() {
        }
    };
}

Example 2 with UnsafeRow

use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.

the class TestSorter method runSorter.

public void runSorter(MyCatMemory myCatMemory, MemoryManager memoryManager, MycatPropertyConf conf) throws NoSuchFieldException, IllegalAccessException, IOException {
    DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId());
    /**
         * 1.schema ,模拟一个field字段值
         *
         */
    int fieldCount = 3;
    ColMeta colMeta = null;
    Map<String, ColMeta> colMetaMap = new HashMap<String, ColMeta>(fieldCount);
    colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("id", colMeta);
    colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("name", colMeta);
    colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING);
    colMetaMap.put("age", colMeta);
    OrderCol[] orderCols = new OrderCol[1];
    OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_ASC);
    orderCols[0] = orderCol;
    /**
         * 2 .PrefixComputer
         */
    StructType schema = new StructType(colMetaMap, fieldCount);
    schema.setOrderCols(orderCols);
    UnsafeExternalRowSorter.PrefixComputer prefixComputer = new RowPrefixComputer(schema);
    /**
         * 3 .PrefixComparator 默认是ASC，可以选择DESC
         */
    final PrefixComparator prefixComparator = PrefixComparators.LONG;
    UnsafeExternalRowSorter sorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "1m"), true, /**使用基数排序？true or false*/
    true);
    UnsafeRow unsafeRow;
    BufferHolder bufferHolder;
    UnsafeRowWriter unsafeRowWriter;
    String line = "testUnsafeRow";
    final Random rand = new Random(42);
    for (int i = 0; i < TEST_SIZE; i++) {
        unsafeRow = new UnsafeRow(3);
        bufferHolder = new BufferHolder(unsafeRow);
        unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 3);
        bufferHolder.reset();
        String key = getRandomString(rand.nextInt(300) + 100);
        unsafeRowWriter.write(0, key.getBytes());
        unsafeRowWriter.write(1, line.getBytes());
        unsafeRowWriter.write(2, ("35" + 1).getBytes());
        unsafeRow.setTotalSize(bufferHolder.totalSize());
        sorter.insertRow(unsafeRow);
    }
    Iterator<UnsafeRow> iter = sorter.sort();
    UnsafeRow row = null;
    int indexprint = 0;
    while (iter.hasNext()) {
        row = iter.next();
        indexprint++;
    }
    sorter.cleanupResources();
    countDownLatch.countDown();
    System.out.println("Thread ID :" + Thread.currentThread().getId() + "Index : " + indexprint);
}

Example 3 with UnsafeRow

use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.

the class UnsafeExternalRowSorter method sort.

public Iterator<UnsafeRow> sort() throws IOException {
    try {
        final UnsafeSorterIterator sortedIterator = sorter.getSortedIterator();
        if (!sortedIterator.hasNext()) {
            cleanupResources();
        }
        return new AbstractScalaRowIterator<UnsafeRow>() {

            private final int numFields = schema.length();

            private UnsafeRow row = new UnsafeRow(numFields);

            @Override
            public boolean hasNext() {
                return sortedIterator.hasNext();
            }

            @Override
            public UnsafeRow next() {
                try {
                    sortedIterator.loadNext();
                    row.pointTo(sortedIterator.getBaseObject(), sortedIterator.getBaseOffset(), sortedIterator.getRecordLength());
                    if (!hasNext()) {
                        // so that we don't have dangling pointers to freed page
                        UnsafeRow copy = row.copy();
                        // so that we don't keep references to the base object
                        row = null;
                        cleanupResources();
                        return copy;
                    } else {
                        return row;
                    }
                } catch (IOException e) {
                    cleanupResources();
                    // Scala iterators don't declare any checked exceptions, so we need to use this hack
                    // to re-throw the exception:
                    Platform.throwException(e);
                }
                throw new RuntimeException("Exception should have been re-thrown in next()");
            }

            @Override
            public void remove() {
            }
        };
    } catch (IOException e) {
        cleanupResources();
        throw e;
    }
}

Example 4 with UnsafeRow

use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.

the class UnsafeExternalRowSorter method mergerSort.

public Iterator<UnsafeRow> mergerSort(List<UnsafeSorterIterator> list) throws IOException {
    UnsafeRowsMerger unsafeRowsMerger = new UnsafeRowsMerger(recordComparator, prefixComparator, list.size());
    for (int i = 0; i < list.size(); i++) {
        unsafeRowsMerger.addSpillIfNotEmpty(list.get(i));
    }
    try {
        final UnsafeSorterIterator sortedIterator = unsafeRowsMerger.getSortedIterator();
        if (!sortedIterator.hasNext()) {
            cleanupResources();
        }
        return new AbstractScalaRowIterator<UnsafeRow>() {

            private final int numFields = schema.length();

            private UnsafeRow row = new UnsafeRow(numFields);

            @Override
            public boolean hasNext() {
                return sortedIterator.hasNext();
            }

            @Override
            public UnsafeRow next() {
                try {
                    sortedIterator.loadNext();
                    row.pointTo(sortedIterator.getBaseObject(), sortedIterator.getBaseOffset(), sortedIterator.getRecordLength());
                    if (!hasNext()) {
                        // so that we don't have dangling pointers to freed page
                        UnsafeRow copy = row.copy();
                        // so that we don't keep references to the base object
                        row = null;
                        cleanupResources();
                        return copy;
                    } else {
                        return row;
                    }
                } catch (IOException e) {
                    cleanupResources();
                    // Scala iterators don't declare any checked exceptions, so we need to use this hack
                    // to re-throw the exception:
                    Platform.throwException(e);
                }
                throw new RuntimeException("Exception should have been re-thrown in next()");
            }

            @Override
            public void remove() {
            }
        };
    } catch (IOException e) {
        cleanupResources();
        throw e;
    }
}

Example 5 with UnsafeRow

use of io.mycat.memory.unsafe.row.UnsafeRow in project Mycat-Server by MyCATApache.

the class UnsafeRowGrouper method getAllBinaryRow.

public UnsafeRow getAllBinaryRow(UnsafeRow row) throws UnsupportedEncodingException {
    UnsafeRow value = new UnsafeRow(this.valuefieldCount);
    bufferHolder = new BufferHolder(value, 0);
    unsafeRowWriter = new UnsafeRowWriter(bufferHolder, this.valuefieldCount);
    bufferHolder.reset();
    ColMeta curColMeta = null;
    for (Map.Entry<String, ColMeta> fieldEntry : columToIndx.entrySet()) {
        curColMeta = fieldEntry.getValue();
        if (!row.isNullAt(curColMeta.colIndex)) {
            switch(curColMeta.colType) {
                case ColMeta.COL_TYPE_BIT:
                    unsafeRowWriter.write(curColMeta.colIndex, row.getByte(curColMeta.colIndex));
                    break;
                case ColMeta.COL_TYPE_INT:
                case ColMeta.COL_TYPE_LONG:
                case ColMeta.COL_TYPE_INT24:
                    unsafeRowWriter.write(curColMeta.colIndex, BytesTools.int2Bytes(row.getInt(curColMeta.colIndex)));
                    break;
                case ColMeta.COL_TYPE_SHORT:
                    unsafeRowWriter.write(curColMeta.colIndex, BytesTools.short2Bytes(row.getShort(curColMeta.colIndex)));
                    break;
                case ColMeta.COL_TYPE_LONGLONG:
                    unsafeRowWriter.write(curColMeta.colIndex, BytesTools.long2Bytes(row.getLong(curColMeta.colIndex)));
                    break;
                case ColMeta.COL_TYPE_FLOAT:
                    unsafeRowWriter.write(curColMeta.colIndex, BytesTools.float2Bytes(row.getFloat(curColMeta.colIndex)));
                    break;
                case ColMeta.COL_TYPE_DOUBLE:
                    unsafeRowWriter.write(curColMeta.colIndex, BytesTools.double2Bytes(row.getDouble(curColMeta.colIndex)));
                    break;
                case ColMeta.COL_TYPE_NEWDECIMAL:
                    int scale = curColMeta.decimals;
                    BigDecimal decimalVal = row.getDecimal(curColMeta.colIndex, scale);
                    unsafeRowWriter.write(curColMeta.colIndex, decimalVal.toString().getBytes());
                    break;
                default:
                    unsafeRowWriter.write(curColMeta.colIndex, row.getBinary(curColMeta.colIndex));
                    break;
            }
        } else {
            unsafeRowWriter.setNullAt(curColMeta.colIndex);
        }
    }
    value.setTotalSize(bufferHolder.totalSize());
    return value;
}